{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "4a350704",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "    导入相关包\n",
    "\"\"\"\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import lightgbm as lgb\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.linear_model import Ridge\n",
    "\n",
    "class SBBTree():\n",
    "    \"\"\"\n",
    "        SBBTree\n",
    "        Stacking,Bootstap,Bagging\n",
    "    \"\"\"\n",
    "    def __init__(\n",
    "                    self, \n",
    "                    params,\n",
    "                    stacking_num,\n",
    "                    bagging_num,\n",
    "                    bagging_test_size,\n",
    "                    num_boost_round,\n",
    "                    early_stopping_rounds\n",
    "                ):\n",
    "        \"\"\"\n",
    "            Initializes the SBBTree.\n",
    "            Args:\n",
    "              params : lgb params.\n",
    "              stacking_num : k_flod stacking.\n",
    "              bagging_num : bootstrap num.\n",
    "              bagging_test_size : bootstrap sample rate.\n",
    "              num_boost_round : boost num.\n",
    "              early_stopping_rounds : early_stopping_rounds.\n",
    "        \"\"\"\n",
    "        self.params = params\n",
    "        self.stacking_num = stacking_num\n",
    "        self.bagging_num = bagging_num\n",
    "        self.bagging_test_size = bagging_test_size\n",
    "        self.num_boost_round = num_boost_round\n",
    "        self.early_stopping_rounds = early_stopping_rounds\n",
    "\n",
    "        self.model = lgb\n",
    "        self.stacking_model = []\n",
    "        self.bagging_model = []\n",
    "\n",
    "    def fit(self, X, y):\n",
    "        \"\"\" fit model. \"\"\"\n",
    "        if self.stacking_num > 1:\n",
    "            layer_train = np.zeros((X.shape[0], 2))\n",
    "            self.SK = StratifiedKFold(n_splits=self.stacking_num, shuffle=True, random_state=1)\n",
    "            for k,(train_index, test_index) in enumerate(self.SK.split(X, y)):\n",
    "                X_train = X[train_index]\n",
    "                y_train = y[train_index]\n",
    "                X_test = X[test_index]\n",
    "                y_test = y[test_index]\n",
    "\n",
    "                lgb_train = lgb.Dataset(X_train, y_train)\n",
    "                lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train)\n",
    "\n",
    "                gbm = lgb.train(self.params,\n",
    "                            lgb_train,\n",
    "                            num_boost_round=self.num_boost_round,\n",
    "                            valid_sets=lgb_eval,\n",
    "                            early_stopping_rounds=self.early_stopping_rounds)\n",
    "\n",
    "                self.stacking_model.append(gbm)\n",
    "\n",
    "                pred_y = gbm.predict(X_test, num_iteration=gbm.best_iteration)\n",
    "                layer_train[test_index, 1] = pred_y\n",
    "\n",
    "            X = np.hstack((X, layer_train[:,1].reshape((-1,1)))) \n",
    "        else:\n",
    "            pass\n",
    "        for bn in range(self.bagging_num):\n",
    "            X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=self.bagging_test_size, random_state=bn)\n",
    "\n",
    "            lgb_train = lgb.Dataset(X_train, y_train)\n",
    "            lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train)\n",
    "\n",
    "            gbm = lgb.train(self.params,\n",
    "                        lgb_train,\n",
    "                        num_boost_round=10000,\n",
    "                        valid_sets=lgb_eval,\n",
    "                        early_stopping_rounds=200)\n",
    "\n",
    "            self.bagging_model.append(gbm)\n",
    "\n",
    "    def predict(self, X_pred):\n",
    "        \"\"\" predict test data. \"\"\"\n",
    "        if self.stacking_num > 1:\n",
    "            test_pred = np.zeros((X_pred.shape[0], self.stacking_num))\n",
    "            for sn,gbm in enumerate(self.stacking_model):\n",
    "                pred = gbm.predict(X_pred, num_iteration=gbm.best_iteration)\n",
    "                test_pred[:, sn] = pred\n",
    "            X_pred = np.hstack((X_pred, test_pred.mean(axis=1).reshape((-1,1))))  \n",
    "        else:\n",
    "            pass \n",
    "        for bn,gbm in enumerate(self.bagging_model):\n",
    "            pred = gbm.predict(X_pred, num_iteration=gbm.best_iteration)\n",
    "            if bn == 0:\n",
    "                pred_out=pred\n",
    "            else:\n",
    "                pred_out+=pred\n",
    "        return pred_out/self.bagging_num"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "23b6ff11",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pre_process\n",
    "#读取相关文件\n",
    "train_path = '../data/train.csv'\n",
    "test_path = '../data/test.csv'\n",
    "train_data = pd.read_csv(train_path)\n",
    "test_data = pd.read_csv(test_path)\n",
    "\n",
    "submit_path = '../data/车辆贷款违约预测挑战赛sample_submit.csv'\n",
    "submit_data = pd.read_csv(submit_path)\n",
    "#生成提交文件\n",
    "submit_data['customer_id'] = test_data['customer_id']\n",
    "submit_data['loan_default'] = 0\n",
    "train_data,test_data = pre_process.fill_inf(train_data,test_data)#填补inf值\n",
    "train_data,test_data = pre_process.del_singular_feature(train_data,test_data)#删除单值属性"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "9559c203",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nan\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\tools\\conda\\envs\\ml\\lib\\site-packages\\ipykernel_launcher.py:27: RuntimeWarning: invalid value encountered in longlong_scalars\n"
     ]
    }
   ],
   "source": [
    "def find_outliers(model,X,y,sigma=3):\n",
    "    # predict y values using model\n",
    "    try:\n",
    "        y_pred = pd.Series(model.predict(X),index=y.index)\n",
    "    # if predicting fails, try fitting the model first\n",
    "    except:\n",
    "        model.fit(X,y)\n",
    "        y_pred = pd.Series(model.predict(X),index=y.index)\n",
    "    \n",
    "    # calculate residuals between the model prediction and true y values\n",
    "    resid = y - y_pred\n",
    "    mean_resid = resid.mean()\n",
    "    std_resid  = resid.std()\n",
    "    \n",
    "    # calculate z statistic, define outliers to be where |z|>sigma\n",
    "    z = (resid-mean_resid)/std_resid\n",
    "    outliers = z[abs(z)>sigma].index\n",
    "    return outliers\n",
    "\n",
    "# 通过岭回归模型找出异常值，并绘制其分布\n",
    "\n",
    "X_train = train_data.iloc[:,0:-1]\n",
    "y_train = train_data.iloc[:,-1]\n",
    "outliers = find_outliers(Ridge(),X_train,y_train)\n",
    "outlier_index = list(outliers)\n",
    "outlier_sum = np.sum(train_data.iloc[outlier_index]['loan_default'].values)\n",
    "print(outlier_sum/len(outlier_index)*100)\n",
    "train_data=train_data.drop(labels=outlier_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "016ad54a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>main_account_loan_no</th>\n",
       "      <th>main_account_active_loan_no</th>\n",
       "      <th>main_account_overdue_no</th>\n",
       "      <th>main_account_outstanding_loan</th>\n",
       "      <th>main_account_sanction_loan</th>\n",
       "      <th>main_account_disbursed_loan</th>\n",
       "      <th>sub_account_loan_no</th>\n",
       "      <th>sub_account_active_loan_no</th>\n",
       "      <th>sub_account_overdue_no</th>\n",
       "      <th>sub_account_outstanding_loan</th>\n",
       "      <th>...</th>\n",
       "      <th>total_monthly_payment</th>\n",
       "      <th>outstanding_disburse_ratio</th>\n",
       "      <th>main_account_tenure</th>\n",
       "      <th>sub_account_tenure</th>\n",
       "      <th>disburse_to_sactioned_ratio</th>\n",
       "      <th>active_to_inactive_act_ratio</th>\n",
       "      <th>Credit_level</th>\n",
       "      <th>employment_type</th>\n",
       "      <th>age</th>\n",
       "      <th>loan_default</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>384989</td>\n",
       "      <td>666207</td>\n",
       "      <td>666207</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>8169</td>\n",
       "      <td>1.73</td>\n",
       "      <td>81</td>\n",
       "      <td>0</td>\n",
       "      <td>1.00</td>\n",
       "      <td>2.50</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>51</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>268670</td>\n",
       "      <td>387994</td>\n",
       "      <td>387994</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>2400</td>\n",
       "      <td>1.44</td>\n",
       "      <td>161</td>\n",
       "      <td>0</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.33</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>27</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>3519013</td>\n",
       "      <td>3613854</td>\n",
       "      <td>3576048</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1.02</td>\n",
       "      <td>3576048</td>\n",
       "      <td>0</td>\n",
       "      <td>0.99</td>\n",
       "      <td>3.00</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>28</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>43</td>\n",
       "      <td>13</td>\n",
       "      <td>6</td>\n",
       "      <td>1867106</td>\n",
       "      <td>2484678</td>\n",
       "      <td>2486856</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>4320912</td>\n",
       "      <td>1.33</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.42</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>55</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>24</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149995</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>33</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149996</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1996</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149997</th>\n",
       "      <td>21</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>60522</td>\n",
       "      <td>119000</td>\n",
       "      <td>119000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>30703</td>\n",
       "      <td>1.97</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.22</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>38</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149998</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>31</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149999</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>31</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>150000 rows × 48 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        main_account_loan_no  main_account_active_loan_no  \\\n",
       "0                          4                            3   \n",
       "1                          7                            2   \n",
       "2                          5                            4   \n",
       "3                         43                           13   \n",
       "4                          0                            0   \n",
       "...                      ...                          ...   \n",
       "149995                     1                            0   \n",
       "149996                     1                            0   \n",
       "149997                    21                            4   \n",
       "149998                     0                            0   \n",
       "149999                     0                            0   \n",
       "\n",
       "        main_account_overdue_no  main_account_outstanding_loan  \\\n",
       "0                             0                         384989   \n",
       "1                             0                         268670   \n",
       "2                             1                        3519013   \n",
       "3                             6                        1867106   \n",
       "4                             0                              0   \n",
       "...                         ...                            ...   \n",
       "149995                        0                              0   \n",
       "149996                        0                              0   \n",
       "149997                        0                          60522   \n",
       "149998                        0                              0   \n",
       "149999                        0                              0   \n",
       "\n",
       "        main_account_sanction_loan  main_account_disbursed_loan  \\\n",
       "0                           666207                       666207   \n",
       "1                           387994                       387994   \n",
       "2                          3613854                      3576048   \n",
       "3                          2484678                      2486856   \n",
       "4                                0                            0   \n",
       "...                            ...                          ...   \n",
       "149995                           0                            0   \n",
       "149996                           0                            0   \n",
       "149997                      119000                       119000   \n",
       "149998                           0                            0   \n",
       "149999                           0                            0   \n",
       "\n",
       "        sub_account_loan_no  sub_account_active_loan_no  \\\n",
       "0                         0                           0   \n",
       "1                         0                           0   \n",
       "2                         0                           0   \n",
       "3                         0                           0   \n",
       "4                         0                           0   \n",
       "...                     ...                         ...   \n",
       "149995                    0                           0   \n",
       "149996                    0                           0   \n",
       "149997                    0                           0   \n",
       "149998                    0                           0   \n",
       "149999                    0                           0   \n",
       "\n",
       "        sub_account_overdue_no  sub_account_outstanding_loan  ...  \\\n",
       "0                            0                             0  ...   \n",
       "1                            0                             0  ...   \n",
       "2                            0                             0  ...   \n",
       "3                            0                             0  ...   \n",
       "4                            0                             0  ...   \n",
       "...                        ...                           ...  ...   \n",
       "149995                       0                             0  ...   \n",
       "149996                       0                             0  ...   \n",
       "149997                       0                             0  ...   \n",
       "149998                       0                             0  ...   \n",
       "149999                       0                             0  ...   \n",
       "\n",
       "        total_monthly_payment  outstanding_disburse_ratio  \\\n",
       "0                        8169                        1.73   \n",
       "1                        2400                        1.44   \n",
       "2                           0                        1.02   \n",
       "3                     4320912                        1.33   \n",
       "4                           0                        1.00   \n",
       "...                       ...                         ...   \n",
       "149995                      0                        1.00   \n",
       "149996                   1996                        1.00   \n",
       "149997                  30703                        1.97   \n",
       "149998                      0                        1.00   \n",
       "149999                      0                        1.00   \n",
       "\n",
       "        main_account_tenure  sub_account_tenure  disburse_to_sactioned_ratio  \\\n",
       "0                        81                   0                         1.00   \n",
       "1                       161                   0                         1.00   \n",
       "2                   3576048                   0                         0.99   \n",
       "3                         0                   0                         1.00   \n",
       "4                         0                   0                         1.00   \n",
       "...                     ...                 ...                          ...   \n",
       "149995                    0                   0                         1.00   \n",
       "149996                    0                   0                         1.00   \n",
       "149997                    3                   0                         1.00   \n",
       "149998                    0                   0                         1.00   \n",
       "149999                    0                   0                         1.00   \n",
       "\n",
       "        active_to_inactive_act_ratio  Credit_level  employment_type  age  \\\n",
       "0                               2.50             1                0   51   \n",
       "1                               1.33             9                0   27   \n",
       "2                               3.00            13                1   28   \n",
       "3                               1.42             3                1   55   \n",
       "4                               1.00            -1                0   24   \n",
       "...                              ...           ...              ...  ...   \n",
       "149995                          1.00            13                0   33   \n",
       "149996                          1.00             6                1   24   \n",
       "149997                          1.22             7                0   38   \n",
       "149998                          1.00            -1                0   31   \n",
       "149999                          1.00            -1                1   31   \n",
       "\n",
       "        loan_default  \n",
       "0                  0  \n",
       "1                  0  \n",
       "2                  0  \n",
       "3                  0  \n",
       "4                  0  \n",
       "...              ...  \n",
       "149995             0  \n",
       "149996             0  \n",
       "149997             0  \n",
       "149998             0  \n",
       "149999             0  \n",
       "\n",
       "[150000 rows x 48 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "09eeb5ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "# from sklearn.feature_selection import VarianceThreshold\n",
    "# from sklearn.datasets import load_iris\n",
    "\n",
    "# #方差选择法，返回值为特征选择后的数据\n",
    "# #参数threshold为方差的阈值\n",
    "# train_data = VarianceThreshold(threshold=3).fit_transform(train_data)\n",
    "# test_data = VarianceThreshold(threshold=3).fit_transform(test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "f2029ff3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import lightgbm as lgb\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "\n",
    "features_columns = [col for col in train_data.columns if col not in ['loan_default']]\n",
    "train = train_data[features_columns].values\n",
    "test = test_data[features_columns].values\n",
    "target =train_data['loan_default'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "f5957de3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.0000e+00, 3.1324e+04, 0.0000e+00, ..., 1.0000e+00, 0.0000e+00,\n",
       "        5.1000e+01],\n",
       "       [0.0000e+00, 5.3078e+04, 1.0000e+00, ..., 1.0000e+00, 0.0000e+00,\n",
       "        2.7000e+01],\n",
       "       [1.0000e+00, 5.3639e+04, 5.6000e+01, ..., 9.9000e-01, 1.0000e+00,\n",
       "        2.8000e+01],\n",
       "       ...,\n",
       "       [0.0000e+00, 5.3278e+04, 1.1000e+01, ..., 1.0000e+00, 0.0000e+00,\n",
       "        3.8000e+01],\n",
       "       [0.0000e+00, 5.9066e+04, 2.6000e+01, ..., 1.0000e+00, 0.0000e+00,\n",
       "        3.1000e+01],\n",
       "       [0.0000e+00, 4.8349e+04, 1.1000e+01, ..., 1.0000e+00, 1.0000e+00,\n",
       "        3.1000e+01]])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# from sklearn.feature_selection import SelectFromModel\n",
    "# from sklearn.ensemble import GradientBoostingClassifier\n",
    " \n",
    "# # GBDT作为基模型的特征选择\n",
    "# SelectFromModel(GradientBoostingClassifier()).fit_transform(train,target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "91e1dde3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Info] Number of positive: 21236, number of negative: 98764\n",
      "[LightGBM] [Info] [cross_entropy:Init]: (metric) labels passed interval [0, 1] check\n",
      "[LightGBM] [Info] [cross_entropy:Init]: sum-of-weights = 120000.000000\n",
      "[LightGBM] [Warning] Auto-choosing row-wise multi-threading, the overhead of testing was 0.021828 seconds.\n",
      "You can set `force_row_wise=true` to remove the overhead.\n",
      "And if memory is not enough, you can set `force_col_wise=true`.\n",
      "[LightGBM] [Info] Total Bins 6595\n",
      "[LightGBM] [Info] Number of data points in the train set: 120000, number of used features: 47\n",
      "[LightGBM] [Info] [cross_entropy:Init]: (metric) labels passed interval [0, 1] check\n",
      "[LightGBM] [Info] [cross_entropy:Init]: sum-of-weights = 30000.000000\n",
      "[LightGBM] [Info] [binary:BoostFromScore]: pavg=0.176967 -> initscore=-1.537035\n",
      "[LightGBM] [Info] Start training from score -1.537035\n",
      "[1]\tvalid_0's cross_entropy: 0.466439\n",
      "Training until validation scores don't improve for 200 rounds\n",
      "[2]\tvalid_0's cross_entropy: 0.466135\n",
      "[3]\tvalid_0's cross_entropy: 0.465871\n",
      "[4]\tvalid_0's cross_entropy: 0.465574\n",
      "[5]\tvalid_0's cross_entropy: 0.465296\n",
      "[6]\tvalid_0's cross_entropy: 0.465046\n",
      "[7]\tvalid_0's cross_entropy: 0.464794\n",
      "[8]\tvalid_0's cross_entropy: 0.464541\n",
      "[9]\tvalid_0's cross_entropy: 0.464289\n",
      "[10]\tvalid_0's cross_entropy: 0.464044\n",
      "[11]\tvalid_0's cross_entropy: 0.463796\n",
      "[12]\tvalid_0's cross_entropy: 0.463562\n",
      "[13]\tvalid_0's cross_entropy: 0.463329\n",
      "[14]\tvalid_0's cross_entropy: 0.463091\n",
      "[15]\tvalid_0's cross_entropy: 0.462871\n",
      "[16]\tvalid_0's cross_entropy: 0.462639\n",
      "[17]\tvalid_0's cross_entropy: 0.462418\n",
      "[18]\tvalid_0's cross_entropy: 0.462198\n",
      "[19]\tvalid_0's cross_entropy: 0.461992\n",
      "[20]\tvalid_0's cross_entropy: 0.461756\n",
      "[21]\tvalid_0's cross_entropy: 0.461537\n",
      "[22]\tvalid_0's cross_entropy: 0.461349\n",
      "[23]\tvalid_0's cross_entropy: 0.461146\n",
      "[24]\tvalid_0's cross_entropy: 0.460934\n",
      "[25]\tvalid_0's cross_entropy: 0.460714\n",
      "[26]\tvalid_0's cross_entropy: 0.460517\n",
      "[27]\tvalid_0's cross_entropy: 0.460312\n",
      "[28]\tvalid_0's cross_entropy: 0.460136\n",
      "[29]\tvalid_0's cross_entropy: 0.45994\n",
      "[30]\tvalid_0's cross_entropy: 0.459749\n",
      "[31]\tvalid_0's cross_entropy: 0.459563\n",
      "[32]\tvalid_0's cross_entropy: 0.459388\n",
      "[33]\tvalid_0's cross_entropy: 0.459214\n",
      "[34]\tvalid_0's cross_entropy: 0.459045\n",
      "[35]\tvalid_0's cross_entropy: 0.458869\n",
      "[36]\tvalid_0's cross_entropy: 0.458699\n",
      "[37]\tvalid_0's cross_entropy: 0.458537\n",
      "[38]\tvalid_0's cross_entropy: 0.458379\n",
      "[39]\tvalid_0's cross_entropy: 0.458224\n",
      "[40]\tvalid_0's cross_entropy: 0.458083\n",
      "[41]\tvalid_0's cross_entropy: 0.45793\n",
      "[42]\tvalid_0's cross_entropy: 0.457788\n",
      "[43]\tvalid_0's cross_entropy: 0.457637\n",
      "[44]\tvalid_0's cross_entropy: 0.457488\n",
      "[45]\tvalid_0's cross_entropy: 0.457363\n",
      "[46]\tvalid_0's cross_entropy: 0.457216\n",
      "[47]\tvalid_0's cross_entropy: 0.457073\n",
      "[48]\tvalid_0's cross_entropy: 0.456926\n",
      "[49]\tvalid_0's cross_entropy: 0.456791\n",
      "[50]\tvalid_0's cross_entropy: 0.456652\n",
      "[51]\tvalid_0's cross_entropy: 0.456502\n",
      "[52]\tvalid_0's cross_entropy: 0.456368\n",
      "[53]\tvalid_0's cross_entropy: 0.456224\n",
      "[54]\tvalid_0's cross_entropy: 0.45609\n",
      "[55]\tvalid_0's cross_entropy: 0.455963\n",
      "[56]\tvalid_0's cross_entropy: 0.455833\n",
      "[57]\tvalid_0's cross_entropy: 0.455705\n",
      "[58]\tvalid_0's cross_entropy: 0.455573\n",
      "[59]\tvalid_0's cross_entropy: 0.455451\n",
      "[60]\tvalid_0's cross_entropy: 0.455323\n",
      "[61]\tvalid_0's cross_entropy: 0.455185\n",
      "[62]\tvalid_0's cross_entropy: 0.45508\n",
      "[63]\tvalid_0's cross_entropy: 0.45494\n",
      "[64]\tvalid_0's cross_entropy: 0.454817\n",
      "[65]\tvalid_0's cross_entropy: 0.454695\n",
      "[66]\tvalid_0's cross_entropy: 0.454585\n",
      "[67]\tvalid_0's cross_entropy: 0.454485\n",
      "[68]\tvalid_0's cross_entropy: 0.454366\n",
      "[69]\tvalid_0's cross_entropy: 0.454263\n",
      "[70]\tvalid_0's cross_entropy: 0.454165\n",
      "[71]\tvalid_0's cross_entropy: 0.454071\n",
      "[72]\tvalid_0's cross_entropy: 0.453956\n",
      "[73]\tvalid_0's cross_entropy: 0.453863\n",
      "[74]\tvalid_0's cross_entropy: 0.453765\n",
      "[75]\tvalid_0's cross_entropy: 0.453668\n",
      "[76]\tvalid_0's cross_entropy: 0.453557\n",
      "[77]\tvalid_0's cross_entropy: 0.453464\n",
      "[78]\tvalid_0's cross_entropy: 0.453376\n",
      "[79]\tvalid_0's cross_entropy: 0.453279\n",
      "[80]\tvalid_0's cross_entropy: 0.453184\n",
      "[81]\tvalid_0's cross_entropy: 0.453101\n",
      "[82]\tvalid_0's cross_entropy: 0.453025\n",
      "[83]\tvalid_0's cross_entropy: 0.452941\n",
      "[84]\tvalid_0's cross_entropy: 0.45285\n",
      "[85]\tvalid_0's cross_entropy: 0.452778\n",
      "[86]\tvalid_0's cross_entropy: 0.452701\n",
      "[87]\tvalid_0's cross_entropy: 0.452605\n",
      "[88]\tvalid_0's cross_entropy: 0.452527\n",
      "[89]\tvalid_0's cross_entropy: 0.452449\n",
      "[90]\tvalid_0's cross_entropy: 0.452374\n",
      "[91]\tvalid_0's cross_entropy: 0.452281\n",
      "[92]\tvalid_0's cross_entropy: 0.45221\n",
      "[93]\tvalid_0's cross_entropy: 0.452136\n",
      "[94]\tvalid_0's cross_entropy: 0.452068\n",
      "[95]\tvalid_0's cross_entropy: 0.451995\n",
      "[96]\tvalid_0's cross_entropy: 0.451936\n",
      "[97]\tvalid_0's cross_entropy: 0.451859\n",
      "[98]\tvalid_0's cross_entropy: 0.451799\n",
      "[99]\tvalid_0's cross_entropy: 0.451707\n",
      "[100]\tvalid_0's cross_entropy: 0.451632\n",
      "[101]\tvalid_0's cross_entropy: 0.451567\n",
      "[102]\tvalid_0's cross_entropy: 0.45151\n",
      "[103]\tvalid_0's cross_entropy: 0.451432\n",
      "[104]\tvalid_0's cross_entropy: 0.451365\n",
      "[105]\tvalid_0's cross_entropy: 0.451302\n",
      "[106]\tvalid_0's cross_entropy: 0.45124\n",
      "[107]\tvalid_0's cross_entropy: 0.451175\n",
      "[108]\tvalid_0's cross_entropy: 0.451105\n",
      "[109]\tvalid_0's cross_entropy: 0.451032\n",
      "[110]\tvalid_0's cross_entropy: 0.450968\n",
      "[111]\tvalid_0's cross_entropy: 0.450908\n",
      "[112]\tvalid_0's cross_entropy: 0.450847\n",
      "[113]\tvalid_0's cross_entropy: 0.450774\n",
      "[114]\tvalid_0's cross_entropy: 0.45071\n",
      "[115]\tvalid_0's cross_entropy: 0.450646\n",
      "[116]\tvalid_0's cross_entropy: 0.450597\n",
      "[117]\tvalid_0's cross_entropy: 0.450545\n",
      "[118]\tvalid_0's cross_entropy: 0.450493\n",
      "[119]\tvalid_0's cross_entropy: 0.450437\n",
      "[120]\tvalid_0's cross_entropy: 0.450371\n",
      "[121]\tvalid_0's cross_entropy: 0.450322\n",
      "[122]\tvalid_0's cross_entropy: 0.450269\n",
      "[123]\tvalid_0's cross_entropy: 0.450215\n",
      "[124]\tvalid_0's cross_entropy: 0.450143\n",
      "[125]\tvalid_0's cross_entropy: 0.450101\n",
      "[126]\tvalid_0's cross_entropy: 0.450038\n",
      "[127]\tvalid_0's cross_entropy: 0.449995\n",
      "[128]\tvalid_0's cross_entropy: 0.449945\n",
      "[129]\tvalid_0's cross_entropy: 0.44989\n",
      "[130]\tvalid_0's cross_entropy: 0.449835\n",
      "[131]\tvalid_0's cross_entropy: 0.449787\n",
      "[132]\tvalid_0's cross_entropy: 0.449749\n",
      "[133]\tvalid_0's cross_entropy: 0.449703\n",
      "[134]\tvalid_0's cross_entropy: 0.449636\n",
      "[135]\tvalid_0's cross_entropy: 0.449587\n",
      "[136]\tvalid_0's cross_entropy: 0.449529\n",
      "[137]\tvalid_0's cross_entropy: 0.449475\n",
      "[138]\tvalid_0's cross_entropy: 0.449432\n",
      "[139]\tvalid_0's cross_entropy: 0.449369\n",
      "[140]\tvalid_0's cross_entropy: 0.449335\n",
      "[141]\tvalid_0's cross_entropy: 0.449284\n",
      "[142]\tvalid_0's cross_entropy: 0.449234\n",
      "[143]\tvalid_0's cross_entropy: 0.449192\n",
      "[144]\tvalid_0's cross_entropy: 0.449141\n",
      "[145]\tvalid_0's cross_entropy: 0.449085\n",
      "[146]\tvalid_0's cross_entropy: 0.449029\n",
      "[147]\tvalid_0's cross_entropy: 0.448988\n",
      "[148]\tvalid_0's cross_entropy: 0.448951\n",
      "[149]\tvalid_0's cross_entropy: 0.448896\n",
      "[150]\tvalid_0's cross_entropy: 0.448858\n",
      "[151]\tvalid_0's cross_entropy: 0.448811\n",
      "[152]\tvalid_0's cross_entropy: 0.448754\n",
      "[153]\tvalid_0's cross_entropy: 0.448715\n",
      "[154]\tvalid_0's cross_entropy: 0.448673\n",
      "[155]\tvalid_0's cross_entropy: 0.44864\n",
      "[156]\tvalid_0's cross_entropy: 0.448592\n",
      "[157]\tvalid_0's cross_entropy: 0.44855\n",
      "[158]\tvalid_0's cross_entropy: 0.448507\n",
      "[159]\tvalid_0's cross_entropy: 0.448465\n",
      "[160]\tvalid_0's cross_entropy: 0.448428\n",
      "[161]\tvalid_0's cross_entropy: 0.448383\n",
      "[162]\tvalid_0's cross_entropy: 0.448337\n",
      "[163]\tvalid_0's cross_entropy: 0.448291\n",
      "[164]\tvalid_0's cross_entropy: 0.448262\n",
      "[165]\tvalid_0's cross_entropy: 0.448229\n",
      "[166]\tvalid_0's cross_entropy: 0.448197\n",
      "[167]\tvalid_0's cross_entropy: 0.448158\n",
      "[168]\tvalid_0's cross_entropy: 0.44812\n",
      "[169]\tvalid_0's cross_entropy: 0.448083\n",
      "[170]\tvalid_0's cross_entropy: 0.448048\n",
      "[171]\tvalid_0's cross_entropy: 0.448011\n",
      "[172]\tvalid_0's cross_entropy: 0.44798\n",
      "[173]\tvalid_0's cross_entropy: 0.447959\n",
      "[174]\tvalid_0's cross_entropy: 0.447917\n",
      "[175]\tvalid_0's cross_entropy: 0.447874\n",
      "[176]\tvalid_0's cross_entropy: 0.447835\n",
      "[177]\tvalid_0's cross_entropy: 0.447804\n",
      "[178]\tvalid_0's cross_entropy: 0.447775\n",
      "[179]\tvalid_0's cross_entropy: 0.447742\n",
      "[180]\tvalid_0's cross_entropy: 0.447718\n",
      "[181]\tvalid_0's cross_entropy: 0.447695\n",
      "[182]\tvalid_0's cross_entropy: 0.447662\n",
      "[183]\tvalid_0's cross_entropy: 0.44763\n",
      "[184]\tvalid_0's cross_entropy: 0.447603\n",
      "[185]\tvalid_0's cross_entropy: 0.44756\n",
      "[186]\tvalid_0's cross_entropy: 0.44753\n",
      "[187]\tvalid_0's cross_entropy: 0.447503\n",
      "[188]\tvalid_0's cross_entropy: 0.447474\n",
      "[189]\tvalid_0's cross_entropy: 0.447458\n",
      "[190]\tvalid_0's cross_entropy: 0.447431\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[191]\tvalid_0's cross_entropy: 0.447392\n",
      "[192]\tvalid_0's cross_entropy: 0.447359\n",
      "[193]\tvalid_0's cross_entropy: 0.447332\n",
      "[194]\tvalid_0's cross_entropy: 0.4473\n",
      "[195]\tvalid_0's cross_entropy: 0.447264\n",
      "[196]\tvalid_0's cross_entropy: 0.44724\n",
      "[197]\tvalid_0's cross_entropy: 0.4472\n",
      "[198]\tvalid_0's cross_entropy: 0.44719\n",
      "[199]\tvalid_0's cross_entropy: 0.447168\n",
      "[200]\tvalid_0's cross_entropy: 0.447144\n",
      "[201]\tvalid_0's cross_entropy: 0.447115\n",
      "[202]\tvalid_0's cross_entropy: 0.447091\n",
      "[203]\tvalid_0's cross_entropy: 0.447061\n",
      "[204]\tvalid_0's cross_entropy: 0.447031\n",
      "[205]\tvalid_0's cross_entropy: 0.447005\n",
      "[206]\tvalid_0's cross_entropy: 0.446981\n",
      "[207]\tvalid_0's cross_entropy: 0.446964\n",
      "[208]\tvalid_0's cross_entropy: 0.446941\n",
      "[209]\tvalid_0's cross_entropy: 0.446916\n",
      "[210]\tvalid_0's cross_entropy: 0.446894\n",
      "[211]\tvalid_0's cross_entropy: 0.44686\n",
      "[212]\tvalid_0's cross_entropy: 0.446844\n",
      "[213]\tvalid_0's cross_entropy: 0.446812\n",
      "[214]\tvalid_0's cross_entropy: 0.446783\n",
      "[215]\tvalid_0's cross_entropy: 0.446747\n",
      "[216]\tvalid_0's cross_entropy: 0.446714\n",
      "[217]\tvalid_0's cross_entropy: 0.446695\n",
      "[218]\tvalid_0's cross_entropy: 0.446665\n",
      "[219]\tvalid_0's cross_entropy: 0.446633\n",
      "[220]\tvalid_0's cross_entropy: 0.446602\n",
      "[221]\tvalid_0's cross_entropy: 0.446576\n",
      "[222]\tvalid_0's cross_entropy: 0.446557\n",
      "[223]\tvalid_0's cross_entropy: 0.446529\n",
      "[224]\tvalid_0's cross_entropy: 0.446505\n",
      "[225]\tvalid_0's cross_entropy: 0.446476\n",
      "[226]\tvalid_0's cross_entropy: 0.446466\n",
      "[227]\tvalid_0's cross_entropy: 0.44644\n",
      "[228]\tvalid_0's cross_entropy: 0.446419\n",
      "[229]\tvalid_0's cross_entropy: 0.446388\n",
      "[230]\tvalid_0's cross_entropy: 0.446361\n",
      "[231]\tvalid_0's cross_entropy: 0.446345\n",
      "[232]\tvalid_0's cross_entropy: 0.446326\n",
      "[233]\tvalid_0's cross_entropy: 0.446293\n",
      "[234]\tvalid_0's cross_entropy: 0.446278\n",
      "[235]\tvalid_0's cross_entropy: 0.44626\n",
      "[236]\tvalid_0's cross_entropy: 0.446237\n",
      "[237]\tvalid_0's cross_entropy: 0.446222\n",
      "[238]\tvalid_0's cross_entropy: 0.446208\n",
      "[239]\tvalid_0's cross_entropy: 0.446191\n",
      "[240]\tvalid_0's cross_entropy: 0.446164\n",
      "[241]\tvalid_0's cross_entropy: 0.446132\n",
      "[242]\tvalid_0's cross_entropy: 0.446108\n",
      "[243]\tvalid_0's cross_entropy: 0.446089\n",
      "[244]\tvalid_0's cross_entropy: 0.446078\n",
      "[245]\tvalid_0's cross_entropy: 0.446056\n",
      "[246]\tvalid_0's cross_entropy: 0.446049\n",
      "[247]\tvalid_0's cross_entropy: 0.446028\n",
      "[248]\tvalid_0's cross_entropy: 0.445995\n",
      "[249]\tvalid_0's cross_entropy: 0.445985\n",
      "[250]\tvalid_0's cross_entropy: 0.445978\n",
      "[251]\tvalid_0's cross_entropy: 0.445956\n",
      "[252]\tvalid_0's cross_entropy: 0.445932\n",
      "[253]\tvalid_0's cross_entropy: 0.445905\n",
      "[254]\tvalid_0's cross_entropy: 0.445902\n",
      "[255]\tvalid_0's cross_entropy: 0.445887\n",
      "[256]\tvalid_0's cross_entropy: 0.445874\n",
      "[257]\tvalid_0's cross_entropy: 0.445858\n",
      "[258]\tvalid_0's cross_entropy: 0.445847\n",
      "[259]\tvalid_0's cross_entropy: 0.445828\n",
      "[260]\tvalid_0's cross_entropy: 0.445808\n",
      "[261]\tvalid_0's cross_entropy: 0.445793\n",
      "[262]\tvalid_0's cross_entropy: 0.445775\n",
      "[263]\tvalid_0's cross_entropy: 0.445764\n",
      "[264]\tvalid_0's cross_entropy: 0.445753\n",
      "[265]\tvalid_0's cross_entropy: 0.445734\n",
      "[266]\tvalid_0's cross_entropy: 0.445712\n",
      "[267]\tvalid_0's cross_entropy: 0.445697\n",
      "[268]\tvalid_0's cross_entropy: 0.445677\n",
      "[269]\tvalid_0's cross_entropy: 0.445659\n",
      "[270]\tvalid_0's cross_entropy: 0.445642\n",
      "[271]\tvalid_0's cross_entropy: 0.445637\n",
      "[272]\tvalid_0's cross_entropy: 0.445603\n",
      "[273]\tvalid_0's cross_entropy: 0.445578\n",
      "[274]\tvalid_0's cross_entropy: 0.445562\n",
      "[275]\tvalid_0's cross_entropy: 0.445556\n",
      "[276]\tvalid_0's cross_entropy: 0.445538\n",
      "[277]\tvalid_0's cross_entropy: 0.445515\n",
      "[278]\tvalid_0's cross_entropy: 0.445495\n",
      "[279]\tvalid_0's cross_entropy: 0.445474\n",
      "[280]\tvalid_0's cross_entropy: 0.445461\n",
      "[281]\tvalid_0's cross_entropy: 0.445445\n",
      "[282]\tvalid_0's cross_entropy: 0.445428\n",
      "[283]\tvalid_0's cross_entropy: 0.445406\n",
      "[284]\tvalid_0's cross_entropy: 0.445404\n",
      "[285]\tvalid_0's cross_entropy: 0.445376\n",
      "[286]\tvalid_0's cross_entropy: 0.445359\n",
      "[287]\tvalid_0's cross_entropy: 0.445332\n",
      "[288]\tvalid_0's cross_entropy: 0.445321\n",
      "[289]\tvalid_0's cross_entropy: 0.445307\n",
      "[290]\tvalid_0's cross_entropy: 0.445287\n",
      "[291]\tvalid_0's cross_entropy: 0.445279\n",
      "[292]\tvalid_0's cross_entropy: 0.445269\n",
      "[293]\tvalid_0's cross_entropy: 0.445256\n",
      "[294]\tvalid_0's cross_entropy: 0.445247\n",
      "[295]\tvalid_0's cross_entropy: 0.445235\n",
      "[296]\tvalid_0's cross_entropy: 0.445214\n",
      "[297]\tvalid_0's cross_entropy: 0.445205\n",
      "[298]\tvalid_0's cross_entropy: 0.445193\n",
      "[299]\tvalid_0's cross_entropy: 0.445175\n",
      "[300]\tvalid_0's cross_entropy: 0.445157\n",
      "[301]\tvalid_0's cross_entropy: 0.445145\n",
      "[302]\tvalid_0's cross_entropy: 0.445135\n",
      "[303]\tvalid_0's cross_entropy: 0.445126\n",
      "[304]\tvalid_0's cross_entropy: 0.445117\n",
      "[305]\tvalid_0's cross_entropy: 0.445104\n",
      "[306]\tvalid_0's cross_entropy: 0.445093\n",
      "[307]\tvalid_0's cross_entropy: 0.44508\n",
      "[308]\tvalid_0's cross_entropy: 0.445069\n",
      "[309]\tvalid_0's cross_entropy: 0.44506\n",
      "[310]\tvalid_0's cross_entropy: 0.445054\n",
      "[311]\tvalid_0's cross_entropy: 0.44505\n",
      "[312]\tvalid_0's cross_entropy: 0.445023\n",
      "[313]\tvalid_0's cross_entropy: 0.445011\n",
      "[314]\tvalid_0's cross_entropy: 0.445007\n",
      "[315]\tvalid_0's cross_entropy: 0.444992\n",
      "[316]\tvalid_0's cross_entropy: 0.444986\n",
      "[317]\tvalid_0's cross_entropy: 0.444983\n",
      "[318]\tvalid_0's cross_entropy: 0.444956\n",
      "[319]\tvalid_0's cross_entropy: 0.444951\n",
      "[320]\tvalid_0's cross_entropy: 0.444949\n",
      "[321]\tvalid_0's cross_entropy: 0.444947\n",
      "[322]\tvalid_0's cross_entropy: 0.444938\n",
      "[323]\tvalid_0's cross_entropy: 0.44493\n",
      "[324]\tvalid_0's cross_entropy: 0.444928\n",
      "[325]\tvalid_0's cross_entropy: 0.444916\n",
      "[326]\tvalid_0's cross_entropy: 0.444908\n",
      "[327]\tvalid_0's cross_entropy: 0.444892\n",
      "[328]\tvalid_0's cross_entropy: 0.44487\n",
      "[329]\tvalid_0's cross_entropy: 0.444859\n",
      "[330]\tvalid_0's cross_entropy: 0.444841\n",
      "[331]\tvalid_0's cross_entropy: 0.444842\n",
      "[332]\tvalid_0's cross_entropy: 0.444827\n",
      "[333]\tvalid_0's cross_entropy: 0.444818\n",
      "[334]\tvalid_0's cross_entropy: 0.444807\n",
      "[335]\tvalid_0's cross_entropy: 0.444794\n",
      "[336]\tvalid_0's cross_entropy: 0.44478\n",
      "[337]\tvalid_0's cross_entropy: 0.444775\n",
      "[338]\tvalid_0's cross_entropy: 0.444765\n",
      "[339]\tvalid_0's cross_entropy: 0.444762\n",
      "[340]\tvalid_0's cross_entropy: 0.444749\n",
      "[341]\tvalid_0's cross_entropy: 0.444749\n",
      "[342]\tvalid_0's cross_entropy: 0.444739\n",
      "[343]\tvalid_0's cross_entropy: 0.444738\n",
      "[344]\tvalid_0's cross_entropy: 0.444725\n",
      "[345]\tvalid_0's cross_entropy: 0.444711\n",
      "[346]\tvalid_0's cross_entropy: 0.444704\n",
      "[347]\tvalid_0's cross_entropy: 0.444699\n",
      "[348]\tvalid_0's cross_entropy: 0.444687\n",
      "[349]\tvalid_0's cross_entropy: 0.444674\n",
      "[350]\tvalid_0's cross_entropy: 0.44466\n",
      "[351]\tvalid_0's cross_entropy: 0.444642\n",
      "[352]\tvalid_0's cross_entropy: 0.444633\n",
      "[353]\tvalid_0's cross_entropy: 0.444625\n",
      "[354]\tvalid_0's cross_entropy: 0.444609\n",
      "[355]\tvalid_0's cross_entropy: 0.444608\n",
      "[356]\tvalid_0's cross_entropy: 0.444596\n",
      "[357]\tvalid_0's cross_entropy: 0.444585\n",
      "[358]\tvalid_0's cross_entropy: 0.44458\n",
      "[359]\tvalid_0's cross_entropy: 0.444572\n",
      "[360]\tvalid_0's cross_entropy: 0.444554\n",
      "[361]\tvalid_0's cross_entropy: 0.444544\n",
      "[362]\tvalid_0's cross_entropy: 0.444544\n",
      "[363]\tvalid_0's cross_entropy: 0.444533\n",
      "[364]\tvalid_0's cross_entropy: 0.444513\n",
      "[365]\tvalid_0's cross_entropy: 0.444517\n",
      "[366]\tvalid_0's cross_entropy: 0.444511\n",
      "[367]\tvalid_0's cross_entropy: 0.444508\n",
      "[368]\tvalid_0's cross_entropy: 0.444503\n",
      "[369]\tvalid_0's cross_entropy: 0.444484\n",
      "[370]\tvalid_0's cross_entropy: 0.444485\n",
      "[371]\tvalid_0's cross_entropy: 0.444479\n",
      "[372]\tvalid_0's cross_entropy: 0.444478\n",
      "[373]\tvalid_0's cross_entropy: 0.444469\n",
      "[374]\tvalid_0's cross_entropy: 0.444459\n",
      "[375]\tvalid_0's cross_entropy: 0.444452\n",
      "[376]\tvalid_0's cross_entropy: 0.444446\n",
      "[377]\tvalid_0's cross_entropy: 0.444439\n",
      "[378]\tvalid_0's cross_entropy: 0.444431\n",
      "[379]\tvalid_0's cross_entropy: 0.444434\n",
      "[380]\tvalid_0's cross_entropy: 0.44442\n",
      "[381]\tvalid_0's cross_entropy: 0.444412\n",
      "[382]\tvalid_0's cross_entropy: 0.444405\n",
      "[383]\tvalid_0's cross_entropy: 0.444401\n",
      "[384]\tvalid_0's cross_entropy: 0.444396\n",
      "[385]\tvalid_0's cross_entropy: 0.444388\n",
      "[386]\tvalid_0's cross_entropy: 0.444382\n",
      "[387]\tvalid_0's cross_entropy: 0.44437\n",
      "[388]\tvalid_0's cross_entropy: 0.444362\n",
      "[389]\tvalid_0's cross_entropy: 0.444356\n",
      "[390]\tvalid_0's cross_entropy: 0.444345\n",
      "[391]\tvalid_0's cross_entropy: 0.444351\n",
      "[392]\tvalid_0's cross_entropy: 0.444341\n",
      "[393]\tvalid_0's cross_entropy: 0.444348\n",
      "[394]\tvalid_0's cross_entropy: 0.444347\n",
      "[395]\tvalid_0's cross_entropy: 0.444343\n",
      "[396]\tvalid_0's cross_entropy: 0.44433\n",
      "[397]\tvalid_0's cross_entropy: 0.444316\n",
      "[398]\tvalid_0's cross_entropy: 0.444318\n",
      "[399]\tvalid_0's cross_entropy: 0.444318\n",
      "[400]\tvalid_0's cross_entropy: 0.44432\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[401]\tvalid_0's cross_entropy: 0.444317\n",
      "[402]\tvalid_0's cross_entropy: 0.444312\n",
      "[403]\tvalid_0's cross_entropy: 0.444314\n",
      "[404]\tvalid_0's cross_entropy: 0.444304\n",
      "[405]\tvalid_0's cross_entropy: 0.444293\n",
      "[406]\tvalid_0's cross_entropy: 0.444292\n",
      "[407]\tvalid_0's cross_entropy: 0.444288\n",
      "[408]\tvalid_0's cross_entropy: 0.444281\n",
      "[409]\tvalid_0's cross_entropy: 0.444275\n",
      "[410]\tvalid_0's cross_entropy: 0.444274\n",
      "[411]\tvalid_0's cross_entropy: 0.444267\n",
      "[412]\tvalid_0's cross_entropy: 0.44427\n",
      "[413]\tvalid_0's cross_entropy: 0.444254\n",
      "[414]\tvalid_0's cross_entropy: 0.444258\n",
      "[415]\tvalid_0's cross_entropy: 0.44426\n",
      "[416]\tvalid_0's cross_entropy: 0.444258\n",
      "[417]\tvalid_0's cross_entropy: 0.444246\n",
      "[418]\tvalid_0's cross_entropy: 0.444233\n",
      "[419]\tvalid_0's cross_entropy: 0.444232\n",
      "[420]\tvalid_0's cross_entropy: 0.44422\n",
      "[421]\tvalid_0's cross_entropy: 0.444215\n",
      "[422]\tvalid_0's cross_entropy: 0.444215\n",
      "[423]\tvalid_0's cross_entropy: 0.444212\n",
      "[424]\tvalid_0's cross_entropy: 0.444218\n",
      "[425]\tvalid_0's cross_entropy: 0.444203\n",
      "[426]\tvalid_0's cross_entropy: 0.444191\n",
      "[427]\tvalid_0's cross_entropy: 0.444193\n",
      "[428]\tvalid_0's cross_entropy: 0.444192\n",
      "[429]\tvalid_0's cross_entropy: 0.444189\n",
      "[430]\tvalid_0's cross_entropy: 0.444186\n",
      "[431]\tvalid_0's cross_entropy: 0.444182\n",
      "[432]\tvalid_0's cross_entropy: 0.444177\n",
      "[433]\tvalid_0's cross_entropy: 0.444162\n",
      "[434]\tvalid_0's cross_entropy: 0.444159\n",
      "[435]\tvalid_0's cross_entropy: 0.444158\n",
      "[436]\tvalid_0's cross_entropy: 0.444161\n",
      "[437]\tvalid_0's cross_entropy: 0.444147\n",
      "[438]\tvalid_0's cross_entropy: 0.444137\n",
      "[439]\tvalid_0's cross_entropy: 0.444137\n",
      "[440]\tvalid_0's cross_entropy: 0.444135\n",
      "[441]\tvalid_0's cross_entropy: 0.444132\n",
      "[442]\tvalid_0's cross_entropy: 0.444131\n",
      "[443]\tvalid_0's cross_entropy: 0.444124\n",
      "[444]\tvalid_0's cross_entropy: 0.444126\n",
      "[445]\tvalid_0's cross_entropy: 0.444132\n",
      "[446]\tvalid_0's cross_entropy: 0.444137\n",
      "[447]\tvalid_0's cross_entropy: 0.444136\n",
      "[448]\tvalid_0's cross_entropy: 0.444134\n",
      "[449]\tvalid_0's cross_entropy: 0.444136\n",
      "[450]\tvalid_0's cross_entropy: 0.444131\n",
      "[451]\tvalid_0's cross_entropy: 0.444135\n",
      "[452]\tvalid_0's cross_entropy: 0.444132\n",
      "[453]\tvalid_0's cross_entropy: 0.444129\n",
      "[454]\tvalid_0's cross_entropy: 0.44413\n",
      "[455]\tvalid_0's cross_entropy: 0.444133\n",
      "[456]\tvalid_0's cross_entropy: 0.444124\n",
      "[457]\tvalid_0's cross_entropy: 0.444121\n",
      "[458]\tvalid_0's cross_entropy: 0.444117\n",
      "[459]\tvalid_0's cross_entropy: 0.444109\n",
      "[460]\tvalid_0's cross_entropy: 0.44411\n",
      "[461]\tvalid_0's cross_entropy: 0.444106\n",
      "[462]\tvalid_0's cross_entropy: 0.444107\n",
      "[463]\tvalid_0's cross_entropy: 0.444102\n",
      "[464]\tvalid_0's cross_entropy: 0.444098\n",
      "[465]\tvalid_0's cross_entropy: 0.444088\n",
      "[466]\tvalid_0's cross_entropy: 0.444093\n",
      "[467]\tvalid_0's cross_entropy: 0.444094\n",
      "[468]\tvalid_0's cross_entropy: 0.444089\n",
      "[469]\tvalid_0's cross_entropy: 0.444089\n",
      "[470]\tvalid_0's cross_entropy: 0.444086\n",
      "[471]\tvalid_0's cross_entropy: 0.444077\n",
      "[472]\tvalid_0's cross_entropy: 0.444076\n",
      "[473]\tvalid_0's cross_entropy: 0.444068\n",
      "[474]\tvalid_0's cross_entropy: 0.444073\n",
      "[475]\tvalid_0's cross_entropy: 0.444071\n",
      "[476]\tvalid_0's cross_entropy: 0.444065\n",
      "[477]\tvalid_0's cross_entropy: 0.444055\n",
      "[478]\tvalid_0's cross_entropy: 0.444054\n",
      "[479]\tvalid_0's cross_entropy: 0.444053\n",
      "[480]\tvalid_0's cross_entropy: 0.444057\n",
      "[481]\tvalid_0's cross_entropy: 0.444051\n",
      "[482]\tvalid_0's cross_entropy: 0.44404\n",
      "[483]\tvalid_0's cross_entropy: 0.44404\n",
      "[484]\tvalid_0's cross_entropy: 0.444038\n",
      "[485]\tvalid_0's cross_entropy: 0.444023\n",
      "[486]\tvalid_0's cross_entropy: 0.444029\n",
      "[487]\tvalid_0's cross_entropy: 0.444029\n",
      "[488]\tvalid_0's cross_entropy: 0.444032\n",
      "[489]\tvalid_0's cross_entropy: 0.444035\n",
      "[490]\tvalid_0's cross_entropy: 0.444033\n",
      "[491]\tvalid_0's cross_entropy: 0.444018\n",
      "[492]\tvalid_0's cross_entropy: 0.444017\n",
      "[493]\tvalid_0's cross_entropy: 0.444021\n",
      "[494]\tvalid_0's cross_entropy: 0.444021\n",
      "[495]\tvalid_0's cross_entropy: 0.444016\n",
      "[496]\tvalid_0's cross_entropy: 0.44402\n",
      "[497]\tvalid_0's cross_entropy: 0.444015\n",
      "[498]\tvalid_0's cross_entropy: 0.443991\n",
      "[499]\tvalid_0's cross_entropy: 0.44399\n",
      "[500]\tvalid_0's cross_entropy: 0.443985\n",
      "[501]\tvalid_0's cross_entropy: 0.443982\n",
      "[502]\tvalid_0's cross_entropy: 0.443976\n",
      "[503]\tvalid_0's cross_entropy: 0.443967\n",
      "[504]\tvalid_0's cross_entropy: 0.443975\n",
      "[505]\tvalid_0's cross_entropy: 0.443973\n",
      "[506]\tvalid_0's cross_entropy: 0.443975\n",
      "[507]\tvalid_0's cross_entropy: 0.443973\n",
      "[508]\tvalid_0's cross_entropy: 0.443979\n",
      "[509]\tvalid_0's cross_entropy: 0.443976\n",
      "[510]\tvalid_0's cross_entropy: 0.443979\n",
      "[511]\tvalid_0's cross_entropy: 0.44398\n",
      "[512]\tvalid_0's cross_entropy: 0.443983\n",
      "[513]\tvalid_0's cross_entropy: 0.443988\n",
      "[514]\tvalid_0's cross_entropy: 0.443989\n",
      "[515]\tvalid_0's cross_entropy: 0.443973\n",
      "[516]\tvalid_0's cross_entropy: 0.443975\n",
      "[517]\tvalid_0's cross_entropy: 0.443975\n",
      "[518]\tvalid_0's cross_entropy: 0.443974\n",
      "[519]\tvalid_0's cross_entropy: 0.443967\n",
      "[520]\tvalid_0's cross_entropy: 0.443967\n",
      "[521]\tvalid_0's cross_entropy: 0.443964\n",
      "[522]\tvalid_0's cross_entropy: 0.44397\n",
      "[523]\tvalid_0's cross_entropy: 0.443968\n",
      "[524]\tvalid_0's cross_entropy: 0.443967\n",
      "[525]\tvalid_0's cross_entropy: 0.443967\n",
      "[526]\tvalid_0's cross_entropy: 0.443962\n",
      "[527]\tvalid_0's cross_entropy: 0.443962\n",
      "[528]\tvalid_0's cross_entropy: 0.443966\n",
      "[529]\tvalid_0's cross_entropy: 0.443963\n",
      "[530]\tvalid_0's cross_entropy: 0.44396\n",
      "[531]\tvalid_0's cross_entropy: 0.443955\n",
      "[532]\tvalid_0's cross_entropy: 0.443946\n",
      "[533]\tvalid_0's cross_entropy: 0.443944\n",
      "[534]\tvalid_0's cross_entropy: 0.443946\n",
      "[535]\tvalid_0's cross_entropy: 0.44395\n",
      "[536]\tvalid_0's cross_entropy: 0.443955\n",
      "[537]\tvalid_0's cross_entropy: 0.443952\n",
      "[538]\tvalid_0's cross_entropy: 0.44395\n",
      "[539]\tvalid_0's cross_entropy: 0.443951\n",
      "[540]\tvalid_0's cross_entropy: 0.443947\n",
      "[541]\tvalid_0's cross_entropy: 0.443944\n",
      "[542]\tvalid_0's cross_entropy: 0.443946\n",
      "[543]\tvalid_0's cross_entropy: 0.443941\n",
      "[544]\tvalid_0's cross_entropy: 0.443935\n",
      "[545]\tvalid_0's cross_entropy: 0.443935\n",
      "[546]\tvalid_0's cross_entropy: 0.443929\n",
      "[547]\tvalid_0's cross_entropy: 0.443928\n",
      "[548]\tvalid_0's cross_entropy: 0.443927\n",
      "[549]\tvalid_0's cross_entropy: 0.443921\n",
      "[550]\tvalid_0's cross_entropy: 0.443919\n",
      "[551]\tvalid_0's cross_entropy: 0.443913\n",
      "[552]\tvalid_0's cross_entropy: 0.443912\n",
      "[553]\tvalid_0's cross_entropy: 0.44391\n",
      "[554]\tvalid_0's cross_entropy: 0.443903\n",
      "[555]\tvalid_0's cross_entropy: 0.4439\n",
      "[556]\tvalid_0's cross_entropy: 0.443901\n",
      "[557]\tvalid_0's cross_entropy: 0.443906\n",
      "[558]\tvalid_0's cross_entropy: 0.443911\n",
      "[559]\tvalid_0's cross_entropy: 0.44391\n",
      "[560]\tvalid_0's cross_entropy: 0.443902\n",
      "[561]\tvalid_0's cross_entropy: 0.443897\n",
      "[562]\tvalid_0's cross_entropy: 0.443897\n",
      "[563]\tvalid_0's cross_entropy: 0.443893\n",
      "[564]\tvalid_0's cross_entropy: 0.443896\n",
      "[565]\tvalid_0's cross_entropy: 0.443895\n",
      "[566]\tvalid_0's cross_entropy: 0.443894\n",
      "[567]\tvalid_0's cross_entropy: 0.4439\n",
      "[568]\tvalid_0's cross_entropy: 0.443892\n",
      "[569]\tvalid_0's cross_entropy: 0.443894\n",
      "[570]\tvalid_0's cross_entropy: 0.443899\n",
      "[571]\tvalid_0's cross_entropy: 0.443898\n",
      "[572]\tvalid_0's cross_entropy: 0.443895\n",
      "[573]\tvalid_0's cross_entropy: 0.443887\n",
      "[574]\tvalid_0's cross_entropy: 0.443885\n",
      "[575]\tvalid_0's cross_entropy: 0.443888\n",
      "[576]\tvalid_0's cross_entropy: 0.443877\n",
      "[577]\tvalid_0's cross_entropy: 0.443877\n",
      "[578]\tvalid_0's cross_entropy: 0.443872\n",
      "[579]\tvalid_0's cross_entropy: 0.443876\n",
      "[580]\tvalid_0's cross_entropy: 0.443881\n",
      "[581]\tvalid_0's cross_entropy: 0.443871\n",
      "[582]\tvalid_0's cross_entropy: 0.443872\n",
      "[583]\tvalid_0's cross_entropy: 0.443868\n",
      "[584]\tvalid_0's cross_entropy: 0.443853\n",
      "[585]\tvalid_0's cross_entropy: 0.443857\n",
      "[586]\tvalid_0's cross_entropy: 0.443864\n",
      "[587]\tvalid_0's cross_entropy: 0.443863\n",
      "[588]\tvalid_0's cross_entropy: 0.443868\n",
      "[589]\tvalid_0's cross_entropy: 0.443857\n",
      "[590]\tvalid_0's cross_entropy: 0.443855\n",
      "[591]\tvalid_0's cross_entropy: 0.443857\n",
      "[592]\tvalid_0's cross_entropy: 0.443852\n",
      "[593]\tvalid_0's cross_entropy: 0.443852\n",
      "[594]\tvalid_0's cross_entropy: 0.443855\n",
      "[595]\tvalid_0's cross_entropy: 0.443856\n",
      "[596]\tvalid_0's cross_entropy: 0.44385\n",
      "[597]\tvalid_0's cross_entropy: 0.443849\n",
      "[598]\tvalid_0's cross_entropy: 0.443855\n",
      "[599]\tvalid_0's cross_entropy: 0.443855\n",
      "[600]\tvalid_0's cross_entropy: 0.443858\n",
      "[601]\tvalid_0's cross_entropy: 0.443862\n",
      "[602]\tvalid_0's cross_entropy: 0.443861\n",
      "[603]\tvalid_0's cross_entropy: 0.443861\n",
      "[604]\tvalid_0's cross_entropy: 0.443855\n",
      "[605]\tvalid_0's cross_entropy: 0.443861\n",
      "[606]\tvalid_0's cross_entropy: 0.443865\n",
      "[607]\tvalid_0's cross_entropy: 0.443863\n",
      "[608]\tvalid_0's cross_entropy: 0.443858\n",
      "[609]\tvalid_0's cross_entropy: 0.443858\n",
      "[610]\tvalid_0's cross_entropy: 0.443865\n",
      "[611]\tvalid_0's cross_entropy: 0.443869\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[612]\tvalid_0's cross_entropy: 0.443855\n",
      "[613]\tvalid_0's cross_entropy: 0.443857\n",
      "[614]\tvalid_0's cross_entropy: 0.443855\n",
      "[615]\tvalid_0's cross_entropy: 0.443854\n",
      "[616]\tvalid_0's cross_entropy: 0.443843\n",
      "[617]\tvalid_0's cross_entropy: 0.443842\n",
      "[618]\tvalid_0's cross_entropy: 0.443832\n",
      "[619]\tvalid_0's cross_entropy: 0.443837\n",
      "[620]\tvalid_0's cross_entropy: 0.443834\n",
      "[621]\tvalid_0's cross_entropy: 0.443841\n",
      "[622]\tvalid_0's cross_entropy: 0.443843\n",
      "[623]\tvalid_0's cross_entropy: 0.443836\n",
      "[624]\tvalid_0's cross_entropy: 0.44383\n",
      "[625]\tvalid_0's cross_entropy: 0.443836\n",
      "[626]\tvalid_0's cross_entropy: 0.44383\n",
      "[627]\tvalid_0's cross_entropy: 0.443824\n",
      "[628]\tvalid_0's cross_entropy: 0.443825\n",
      "[629]\tvalid_0's cross_entropy: 0.44383\n",
      "[630]\tvalid_0's cross_entropy: 0.443834\n",
      "[631]\tvalid_0's cross_entropy: 0.443829\n",
      "[632]\tvalid_0's cross_entropy: 0.443829\n",
      "[633]\tvalid_0's cross_entropy: 0.443832\n",
      "[634]\tvalid_0's cross_entropy: 0.443832\n",
      "[635]\tvalid_0's cross_entropy: 0.443832\n",
      "[636]\tvalid_0's cross_entropy: 0.443833\n",
      "[637]\tvalid_0's cross_entropy: 0.443828\n",
      "[638]\tvalid_0's cross_entropy: 0.443825\n",
      "[639]\tvalid_0's cross_entropy: 0.44382\n",
      "[640]\tvalid_0's cross_entropy: 0.443823\n",
      "[641]\tvalid_0's cross_entropy: 0.443819\n",
      "[642]\tvalid_0's cross_entropy: 0.443819\n",
      "[643]\tvalid_0's cross_entropy: 0.443821\n",
      "[644]\tvalid_0's cross_entropy: 0.443827\n",
      "[645]\tvalid_0's cross_entropy: 0.443825\n",
      "[646]\tvalid_0's cross_entropy: 0.443825\n",
      "[647]\tvalid_0's cross_entropy: 0.443823\n",
      "[648]\tvalid_0's cross_entropy: 0.443825\n",
      "[649]\tvalid_0's cross_entropy: 0.443827\n",
      "[650]\tvalid_0's cross_entropy: 0.443828\n",
      "[651]\tvalid_0's cross_entropy: 0.443822\n",
      "[652]\tvalid_0's cross_entropy: 0.443825\n",
      "[653]\tvalid_0's cross_entropy: 0.44382\n",
      "[654]\tvalid_0's cross_entropy: 0.443819\n",
      "[655]\tvalid_0's cross_entropy: 0.443817\n",
      "[656]\tvalid_0's cross_entropy: 0.443816\n",
      "[657]\tvalid_0's cross_entropy: 0.443821\n",
      "[658]\tvalid_0's cross_entropy: 0.443824\n",
      "[659]\tvalid_0's cross_entropy: 0.443822\n",
      "[660]\tvalid_0's cross_entropy: 0.44381\n",
      "[661]\tvalid_0's cross_entropy: 0.443803\n",
      "[662]\tvalid_0's cross_entropy: 0.443804\n",
      "[663]\tvalid_0's cross_entropy: 0.443797\n",
      "[664]\tvalid_0's cross_entropy: 0.443804\n",
      "[665]\tvalid_0's cross_entropy: 0.443806\n",
      "[666]\tvalid_0's cross_entropy: 0.443801\n",
      "[667]\tvalid_0's cross_entropy: 0.443805\n",
      "[668]\tvalid_0's cross_entropy: 0.443813\n",
      "[669]\tvalid_0's cross_entropy: 0.443811\n",
      "[670]\tvalid_0's cross_entropy: 0.443813\n",
      "[671]\tvalid_0's cross_entropy: 0.443821\n",
      "[672]\tvalid_0's cross_entropy: 0.443819\n",
      "[673]\tvalid_0's cross_entropy: 0.443804\n",
      "[674]\tvalid_0's cross_entropy: 0.443806\n",
      "[675]\tvalid_0's cross_entropy: 0.44381\n",
      "[676]\tvalid_0's cross_entropy: 0.443805\n",
      "[677]\tvalid_0's cross_entropy: 0.443792\n",
      "[678]\tvalid_0's cross_entropy: 0.443794\n",
      "[679]\tvalid_0's cross_entropy: 0.443795\n",
      "[680]\tvalid_0's cross_entropy: 0.443794\n",
      "[681]\tvalid_0's cross_entropy: 0.443797\n",
      "[682]\tvalid_0's cross_entropy: 0.443791\n",
      "[683]\tvalid_0's cross_entropy: 0.443797\n",
      "[684]\tvalid_0's cross_entropy: 0.443799\n",
      "[685]\tvalid_0's cross_entropy: 0.443799\n",
      "[686]\tvalid_0's cross_entropy: 0.443799\n",
      "[687]\tvalid_0's cross_entropy: 0.443798\n",
      "[688]\tvalid_0's cross_entropy: 0.4438\n",
      "[689]\tvalid_0's cross_entropy: 0.443805\n",
      "[690]\tvalid_0's cross_entropy: 0.443788\n",
      "[691]\tvalid_0's cross_entropy: 0.443782\n",
      "[692]\tvalid_0's cross_entropy: 0.443791\n",
      "[693]\tvalid_0's cross_entropy: 0.443793\n",
      "[694]\tvalid_0's cross_entropy: 0.443802\n",
      "[695]\tvalid_0's cross_entropy: 0.443794\n",
      "[696]\tvalid_0's cross_entropy: 0.443792\n",
      "[697]\tvalid_0's cross_entropy: 0.443794\n",
      "[698]\tvalid_0's cross_entropy: 0.443791\n",
      "[699]\tvalid_0's cross_entropy: 0.443789\n",
      "[700]\tvalid_0's cross_entropy: 0.443786\n",
      "[701]\tvalid_0's cross_entropy: 0.443791\n",
      "[702]\tvalid_0's cross_entropy: 0.443789\n",
      "[703]\tvalid_0's cross_entropy: 0.443787\n",
      "[704]\tvalid_0's cross_entropy: 0.44379\n",
      "[705]\tvalid_0's cross_entropy: 0.443794\n",
      "[706]\tvalid_0's cross_entropy: 0.443792\n",
      "[707]\tvalid_0's cross_entropy: 0.443781\n",
      "[708]\tvalid_0's cross_entropy: 0.443783\n",
      "[709]\tvalid_0's cross_entropy: 0.443774\n",
      "[710]\tvalid_0's cross_entropy: 0.443772\n",
      "[711]\tvalid_0's cross_entropy: 0.443767\n",
      "[712]\tvalid_0's cross_entropy: 0.443768\n",
      "[713]\tvalid_0's cross_entropy: 0.443776\n",
      "[714]\tvalid_0's cross_entropy: 0.443777\n",
      "[715]\tvalid_0's cross_entropy: 0.443775\n",
      "[716]\tvalid_0's cross_entropy: 0.44378\n",
      "[717]\tvalid_0's cross_entropy: 0.443784\n",
      "[718]\tvalid_0's cross_entropy: 0.443786\n",
      "[719]\tvalid_0's cross_entropy: 0.443797\n",
      "[720]\tvalid_0's cross_entropy: 0.443796\n",
      "[721]\tvalid_0's cross_entropy: 0.443794\n",
      "[722]\tvalid_0's cross_entropy: 0.443787\n",
      "[723]\tvalid_0's cross_entropy: 0.443786\n",
      "[724]\tvalid_0's cross_entropy: 0.443787\n",
      "[725]\tvalid_0's cross_entropy: 0.443794\n",
      "[726]\tvalid_0's cross_entropy: 0.443793\n",
      "[727]\tvalid_0's cross_entropy: 0.443795\n",
      "[728]\tvalid_0's cross_entropy: 0.443796\n",
      "[729]\tvalid_0's cross_entropy: 0.443798\n",
      "[730]\tvalid_0's cross_entropy: 0.443799\n",
      "[731]\tvalid_0's cross_entropy: 0.443799\n",
      "[732]\tvalid_0's cross_entropy: 0.443802\n",
      "[733]\tvalid_0's cross_entropy: 0.443803\n",
      "[734]\tvalid_0's cross_entropy: 0.443799\n",
      "[735]\tvalid_0's cross_entropy: 0.443807\n",
      "[736]\tvalid_0's cross_entropy: 0.443804\n",
      "[737]\tvalid_0's cross_entropy: 0.443808\n",
      "[738]\tvalid_0's cross_entropy: 0.443808\n",
      "[739]\tvalid_0's cross_entropy: 0.443806\n",
      "[740]\tvalid_0's cross_entropy: 0.443803\n",
      "[741]\tvalid_0's cross_entropy: 0.4438\n",
      "[742]\tvalid_0's cross_entropy: 0.443794\n",
      "[743]\tvalid_0's cross_entropy: 0.443799\n",
      "[744]\tvalid_0's cross_entropy: 0.443798\n",
      "[745]\tvalid_0's cross_entropy: 0.443804\n",
      "[746]\tvalid_0's cross_entropy: 0.443803\n",
      "[747]\tvalid_0's cross_entropy: 0.443803\n",
      "[748]\tvalid_0's cross_entropy: 0.443796\n",
      "[749]\tvalid_0's cross_entropy: 0.443798\n",
      "[750]\tvalid_0's cross_entropy: 0.443802\n",
      "[751]\tvalid_0's cross_entropy: 0.443806\n",
      "[752]\tvalid_0's cross_entropy: 0.443804\n",
      "[753]\tvalid_0's cross_entropy: 0.443802\n",
      "[754]\tvalid_0's cross_entropy: 0.443804\n",
      "[755]\tvalid_0's cross_entropy: 0.443802\n",
      "[756]\tvalid_0's cross_entropy: 0.4438\n",
      "[757]\tvalid_0's cross_entropy: 0.443791\n",
      "[758]\tvalid_0's cross_entropy: 0.443798\n",
      "[759]\tvalid_0's cross_entropy: 0.443795\n",
      "[760]\tvalid_0's cross_entropy: 0.443787\n",
      "[761]\tvalid_0's cross_entropy: 0.443785\n",
      "[762]\tvalid_0's cross_entropy: 0.443797\n",
      "[763]\tvalid_0's cross_entropy: 0.443795\n",
      "[764]\tvalid_0's cross_entropy: 0.443793\n",
      "[765]\tvalid_0's cross_entropy: 0.443802\n",
      "[766]\tvalid_0's cross_entropy: 0.443799\n",
      "[767]\tvalid_0's cross_entropy: 0.443801\n",
      "[768]\tvalid_0's cross_entropy: 0.443815\n",
      "[769]\tvalid_0's cross_entropy: 0.443814\n",
      "[770]\tvalid_0's cross_entropy: 0.443817\n",
      "[771]\tvalid_0's cross_entropy: 0.443816\n",
      "[772]\tvalid_0's cross_entropy: 0.44382\n",
      "[773]\tvalid_0's cross_entropy: 0.443827\n",
      "[774]\tvalid_0's cross_entropy: 0.443833\n",
      "[775]\tvalid_0's cross_entropy: 0.443836\n",
      "[776]\tvalid_0's cross_entropy: 0.443837\n",
      "[777]\tvalid_0's cross_entropy: 0.443839\n",
      "[778]\tvalid_0's cross_entropy: 0.443843\n",
      "[779]\tvalid_0's cross_entropy: 0.443843\n",
      "[780]\tvalid_0's cross_entropy: 0.443839\n",
      "[781]\tvalid_0's cross_entropy: 0.443841\n",
      "[782]\tvalid_0's cross_entropy: 0.443847\n",
      "[783]\tvalid_0's cross_entropy: 0.443852\n",
      "[784]\tvalid_0's cross_entropy: 0.443855\n",
      "[785]\tvalid_0's cross_entropy: 0.443857\n",
      "[786]\tvalid_0's cross_entropy: 0.443855\n",
      "[787]\tvalid_0's cross_entropy: 0.44386\n",
      "[788]\tvalid_0's cross_entropy: 0.443859\n",
      "[789]\tvalid_0's cross_entropy: 0.443855\n",
      "[790]\tvalid_0's cross_entropy: 0.443851\n",
      "[791]\tvalid_0's cross_entropy: 0.443849\n",
      "[792]\tvalid_0's cross_entropy: 0.443852\n",
      "[793]\tvalid_0's cross_entropy: 0.443857\n",
      "[794]\tvalid_0's cross_entropy: 0.443856\n",
      "[795]\tvalid_0's cross_entropy: 0.443849\n",
      "[796]\tvalid_0's cross_entropy: 0.443848\n",
      "[797]\tvalid_0's cross_entropy: 0.443853\n",
      "[798]\tvalid_0's cross_entropy: 0.443852\n",
      "[799]\tvalid_0's cross_entropy: 0.443857\n",
      "[800]\tvalid_0's cross_entropy: 0.443853\n",
      "[801]\tvalid_0's cross_entropy: 0.44386\n",
      "[802]\tvalid_0's cross_entropy: 0.443856\n",
      "[803]\tvalid_0's cross_entropy: 0.443864\n",
      "[804]\tvalid_0's cross_entropy: 0.443861\n",
      "[805]\tvalid_0's cross_entropy: 0.443859\n",
      "[806]\tvalid_0's cross_entropy: 0.443862\n",
      "[807]\tvalid_0's cross_entropy: 0.443861\n",
      "[808]\tvalid_0's cross_entropy: 0.443861\n",
      "[809]\tvalid_0's cross_entropy: 0.44387\n",
      "[810]\tvalid_0's cross_entropy: 0.443867\n",
      "[811]\tvalid_0's cross_entropy: 0.443867\n",
      "[812]\tvalid_0's cross_entropy: 0.443864\n",
      "[813]\tvalid_0's cross_entropy: 0.443865\n",
      "[814]\tvalid_0's cross_entropy: 0.443869\n",
      "[815]\tvalid_0's cross_entropy: 0.443871\n",
      "[816]\tvalid_0's cross_entropy: 0.443869\n",
      "[817]\tvalid_0's cross_entropy: 0.44387\n",
      "[818]\tvalid_0's cross_entropy: 0.443867\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[819]\tvalid_0's cross_entropy: 0.443872\n",
      "[820]\tvalid_0's cross_entropy: 0.443882\n",
      "[821]\tvalid_0's cross_entropy: 0.443889\n",
      "[822]\tvalid_0's cross_entropy: 0.44389\n",
      "[823]\tvalid_0's cross_entropy: 0.443892\n",
      "[824]\tvalid_0's cross_entropy: 0.443899\n",
      "[825]\tvalid_0's cross_entropy: 0.443901\n",
      "[826]\tvalid_0's cross_entropy: 0.443905\n",
      "[827]\tvalid_0's cross_entropy: 0.443907\n",
      "[828]\tvalid_0's cross_entropy: 0.443911\n",
      "[829]\tvalid_0's cross_entropy: 0.443915\n",
      "[830]\tvalid_0's cross_entropy: 0.443913\n",
      "[831]\tvalid_0's cross_entropy: 0.443913\n",
      "[832]\tvalid_0's cross_entropy: 0.443913\n",
      "[833]\tvalid_0's cross_entropy: 0.443917\n",
      "[834]\tvalid_0's cross_entropy: 0.443916\n",
      "[835]\tvalid_0's cross_entropy: 0.443913\n",
      "[836]\tvalid_0's cross_entropy: 0.443917\n",
      "[837]\tvalid_0's cross_entropy: 0.443923\n",
      "[838]\tvalid_0's cross_entropy: 0.443921\n",
      "[839]\tvalid_0's cross_entropy: 0.443913\n",
      "[840]\tvalid_0's cross_entropy: 0.443913\n",
      "[841]\tvalid_0's cross_entropy: 0.443915\n",
      "[842]\tvalid_0's cross_entropy: 0.443917\n",
      "[843]\tvalid_0's cross_entropy: 0.443917\n",
      "[844]\tvalid_0's cross_entropy: 0.44392\n",
      "[845]\tvalid_0's cross_entropy: 0.443923\n",
      "[846]\tvalid_0's cross_entropy: 0.443928\n",
      "[847]\tvalid_0's cross_entropy: 0.443931\n",
      "[848]\tvalid_0's cross_entropy: 0.443936\n",
      "[849]\tvalid_0's cross_entropy: 0.443928\n",
      "[850]\tvalid_0's cross_entropy: 0.443928\n",
      "[851]\tvalid_0's cross_entropy: 0.443928\n",
      "[852]\tvalid_0's cross_entropy: 0.44393\n",
      "[853]\tvalid_0's cross_entropy: 0.443927\n",
      "[854]\tvalid_0's cross_entropy: 0.443933\n",
      "[855]\tvalid_0's cross_entropy: 0.443931\n",
      "[856]\tvalid_0's cross_entropy: 0.44393\n",
      "[857]\tvalid_0's cross_entropy: 0.443934\n",
      "[858]\tvalid_0's cross_entropy: 0.443935\n",
      "[859]\tvalid_0's cross_entropy: 0.443935\n",
      "[860]\tvalid_0's cross_entropy: 0.443935\n",
      "[861]\tvalid_0's cross_entropy: 0.443936\n",
      "[862]\tvalid_0's cross_entropy: 0.443946\n",
      "[863]\tvalid_0's cross_entropy: 0.443947\n",
      "[864]\tvalid_0's cross_entropy: 0.443952\n",
      "[865]\tvalid_0's cross_entropy: 0.443948\n",
      "[866]\tvalid_0's cross_entropy: 0.443947\n",
      "[867]\tvalid_0's cross_entropy: 0.443946\n",
      "[868]\tvalid_0's cross_entropy: 0.443946\n",
      "[869]\tvalid_0's cross_entropy: 0.443951\n",
      "[870]\tvalid_0's cross_entropy: 0.443954\n",
      "[871]\tvalid_0's cross_entropy: 0.443961\n",
      "[872]\tvalid_0's cross_entropy: 0.443964\n",
      "[873]\tvalid_0's cross_entropy: 0.443966\n",
      "[874]\tvalid_0's cross_entropy: 0.443967\n",
      "[875]\tvalid_0's cross_entropy: 0.443966\n",
      "[876]\tvalid_0's cross_entropy: 0.443965\n",
      "[877]\tvalid_0's cross_entropy: 0.44397\n",
      "[878]\tvalid_0's cross_entropy: 0.443967\n",
      "[879]\tvalid_0's cross_entropy: 0.443966\n",
      "[880]\tvalid_0's cross_entropy: 0.443954\n",
      "[881]\tvalid_0's cross_entropy: 0.443954\n",
      "[882]\tvalid_0's cross_entropy: 0.443953\n",
      "[883]\tvalid_0's cross_entropy: 0.443953\n",
      "[884]\tvalid_0's cross_entropy: 0.443954\n",
      "[885]\tvalid_0's cross_entropy: 0.443956\n",
      "[886]\tvalid_0's cross_entropy: 0.443956\n",
      "[887]\tvalid_0's cross_entropy: 0.44396\n",
      "[888]\tvalid_0's cross_entropy: 0.443961\n",
      "[889]\tvalid_0's cross_entropy: 0.443959\n",
      "[890]\tvalid_0's cross_entropy: 0.443958\n",
      "[891]\tvalid_0's cross_entropy: 0.44395\n",
      "[892]\tvalid_0's cross_entropy: 0.443953\n",
      "[893]\tvalid_0's cross_entropy: 0.443953\n",
      "[894]\tvalid_0's cross_entropy: 0.443951\n",
      "[895]\tvalid_0's cross_entropy: 0.44395\n",
      "[896]\tvalid_0's cross_entropy: 0.443949\n",
      "[897]\tvalid_0's cross_entropy: 0.443953\n",
      "[898]\tvalid_0's cross_entropy: 0.443958\n",
      "[899]\tvalid_0's cross_entropy: 0.443955\n",
      "[900]\tvalid_0's cross_entropy: 0.443954\n",
      "[901]\tvalid_0's cross_entropy: 0.443954\n",
      "[902]\tvalid_0's cross_entropy: 0.443956\n",
      "[903]\tvalid_0's cross_entropy: 0.443959\n",
      "[904]\tvalid_0's cross_entropy: 0.443957\n",
      "[905]\tvalid_0's cross_entropy: 0.443957\n",
      "[906]\tvalid_0's cross_entropy: 0.443959\n",
      "[907]\tvalid_0's cross_entropy: 0.443952\n",
      "[908]\tvalid_0's cross_entropy: 0.443955\n",
      "[909]\tvalid_0's cross_entropy: 0.443953\n",
      "[910]\tvalid_0's cross_entropy: 0.443953\n",
      "[911]\tvalid_0's cross_entropy: 0.443957\n",
      "Early stopping, best iteration is:\n",
      "[711]\tvalid_0's cross_entropy: 0.443767\n",
      "[LightGBM] [Info] Number of positive: 21236, number of negative: 98764\n",
      "[LightGBM] [Info] [cross_entropy:Init]: (metric) labels passed interval [0, 1] check\n",
      "[LightGBM] [Info] [cross_entropy:Init]: sum-of-weights = 120000.000000\n",
      "[LightGBM] [Warning] Auto-choosing row-wise multi-threading, the overhead of testing was 0.023446 seconds.\n",
      "You can set `force_row_wise=true` to remove the overhead.\n",
      "And if memory is not enough, you can set `force_col_wise=true`.\n",
      "[LightGBM] [Info] Total Bins 6606\n",
      "[LightGBM] [Info] Number of data points in the train set: 120000, number of used features: 47\n",
      "[LightGBM] [Info] [cross_entropy:Init]: (metric) labels passed interval [0, 1] check\n",
      "[LightGBM] [Info] [cross_entropy:Init]: sum-of-weights = 30000.000000\n",
      "[LightGBM] [Info] [binary:BoostFromScore]: pavg=0.176967 -> initscore=-1.537035\n",
      "[LightGBM] [Info] Start training from score -1.537035\n",
      "[1]\tvalid_0's cross_entropy: 0.466396\n",
      "Training until validation scores don't improve for 200 rounds\n",
      "[2]\tvalid_0's cross_entropy: 0.466051\n",
      "[3]\tvalid_0's cross_entropy: 0.465754\n",
      "[4]\tvalid_0's cross_entropy: 0.465434\n",
      "[5]\tvalid_0's cross_entropy: 0.46511\n",
      "[6]\tvalid_0's cross_entropy: 0.464823\n",
      "[7]\tvalid_0's cross_entropy: 0.464544\n",
      "[8]\tvalid_0's cross_entropy: 0.464241\n",
      "[9]\tvalid_0's cross_entropy: 0.463949\n",
      "[10]\tvalid_0's cross_entropy: 0.463657\n",
      "[11]\tvalid_0's cross_entropy: 0.463378\n",
      "[12]\tvalid_0's cross_entropy: 0.46311\n",
      "[13]\tvalid_0's cross_entropy: 0.46283\n",
      "[14]\tvalid_0's cross_entropy: 0.462573\n",
      "[15]\tvalid_0's cross_entropy: 0.462331\n",
      "[16]\tvalid_0's cross_entropy: 0.46206\n",
      "[17]\tvalid_0's cross_entropy: 0.461806\n",
      "[18]\tvalid_0's cross_entropy: 0.461554\n",
      "[19]\tvalid_0's cross_entropy: 0.461331\n",
      "[20]\tvalid_0's cross_entropy: 0.461093\n",
      "[21]\tvalid_0's cross_entropy: 0.460857\n",
      "[22]\tvalid_0's cross_entropy: 0.460645\n",
      "[23]\tvalid_0's cross_entropy: 0.460405\n",
      "[24]\tvalid_0's cross_entropy: 0.46018\n",
      "[25]\tvalid_0's cross_entropy: 0.459967\n",
      "[26]\tvalid_0's cross_entropy: 0.459736\n",
      "[27]\tvalid_0's cross_entropy: 0.459518\n",
      "[28]\tvalid_0's cross_entropy: 0.459305\n",
      "[29]\tvalid_0's cross_entropy: 0.459102\n",
      "[30]\tvalid_0's cross_entropy: 0.458882\n",
      "[31]\tvalid_0's cross_entropy: 0.458679\n",
      "[32]\tvalid_0's cross_entropy: 0.45848\n",
      "[33]\tvalid_0's cross_entropy: 0.458288\n",
      "[34]\tvalid_0's cross_entropy: 0.458101\n",
      "[35]\tvalid_0's cross_entropy: 0.457906\n",
      "[36]\tvalid_0's cross_entropy: 0.457726\n",
      "[37]\tvalid_0's cross_entropy: 0.45754\n",
      "[38]\tvalid_0's cross_entropy: 0.457349\n",
      "[39]\tvalid_0's cross_entropy: 0.457183\n",
      "[40]\tvalid_0's cross_entropy: 0.456999\n",
      "[41]\tvalid_0's cross_entropy: 0.456812\n",
      "[42]\tvalid_0's cross_entropy: 0.456632\n",
      "[43]\tvalid_0's cross_entropy: 0.456464\n",
      "[44]\tvalid_0's cross_entropy: 0.456284\n",
      "[45]\tvalid_0's cross_entropy: 0.456119\n",
      "[46]\tvalid_0's cross_entropy: 0.455953\n",
      "[47]\tvalid_0's cross_entropy: 0.455792\n",
      "[48]\tvalid_0's cross_entropy: 0.455621\n",
      "[49]\tvalid_0's cross_entropy: 0.455471\n",
      "[50]\tvalid_0's cross_entropy: 0.455325\n",
      "[51]\tvalid_0's cross_entropy: 0.455179\n",
      "[52]\tvalid_0's cross_entropy: 0.455034\n",
      "[53]\tvalid_0's cross_entropy: 0.454885\n",
      "[54]\tvalid_0's cross_entropy: 0.454729\n",
      "[55]\tvalid_0's cross_entropy: 0.454585\n",
      "[56]\tvalid_0's cross_entropy: 0.454436\n",
      "[57]\tvalid_0's cross_entropy: 0.454306\n",
      "[58]\tvalid_0's cross_entropy: 0.454158\n",
      "[59]\tvalid_0's cross_entropy: 0.454012\n",
      "[60]\tvalid_0's cross_entropy: 0.453875\n",
      "[61]\tvalid_0's cross_entropy: 0.453741\n",
      "[62]\tvalid_0's cross_entropy: 0.453612\n",
      "[63]\tvalid_0's cross_entropy: 0.453484\n",
      "[64]\tvalid_0's cross_entropy: 0.453361\n",
      "[65]\tvalid_0's cross_entropy: 0.453234\n",
      "[66]\tvalid_0's cross_entropy: 0.453106\n",
      "[67]\tvalid_0's cross_entropy: 0.45299\n",
      "[68]\tvalid_0's cross_entropy: 0.452859\n",
      "[69]\tvalid_0's cross_entropy: 0.452738\n",
      "[70]\tvalid_0's cross_entropy: 0.452628\n",
      "[71]\tvalid_0's cross_entropy: 0.4525\n",
      "[72]\tvalid_0's cross_entropy: 0.452396\n",
      "[73]\tvalid_0's cross_entropy: 0.452284\n",
      "[74]\tvalid_0's cross_entropy: 0.452176\n",
      "[75]\tvalid_0's cross_entropy: 0.452069\n",
      "[76]\tvalid_0's cross_entropy: 0.45195\n",
      "[77]\tvalid_0's cross_entropy: 0.451856\n",
      "[78]\tvalid_0's cross_entropy: 0.451737\n",
      "[79]\tvalid_0's cross_entropy: 0.451628\n",
      "[80]\tvalid_0's cross_entropy: 0.451518\n",
      "[81]\tvalid_0's cross_entropy: 0.451415\n",
      "[82]\tvalid_0's cross_entropy: 0.451296\n",
      "[83]\tvalid_0's cross_entropy: 0.451196\n",
      "[84]\tvalid_0's cross_entropy: 0.451091\n",
      "[85]\tvalid_0's cross_entropy: 0.450988\n",
      "[86]\tvalid_0's cross_entropy: 0.450878\n",
      "[87]\tvalid_0's cross_entropy: 0.450768\n",
      "[88]\tvalid_0's cross_entropy: 0.450676\n",
      "[89]\tvalid_0's cross_entropy: 0.450595\n",
      "[90]\tvalid_0's cross_entropy: 0.450506\n",
      "[91]\tvalid_0's cross_entropy: 0.450413\n",
      "[92]\tvalid_0's cross_entropy: 0.450319\n",
      "[93]\tvalid_0's cross_entropy: 0.450227\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[94]\tvalid_0's cross_entropy: 0.450142\n",
      "[95]\tvalid_0's cross_entropy: 0.450061\n",
      "[96]\tvalid_0's cross_entropy: 0.449972\n",
      "[97]\tvalid_0's cross_entropy: 0.449876\n",
      "[98]\tvalid_0's cross_entropy: 0.449786\n",
      "[99]\tvalid_0's cross_entropy: 0.449707\n",
      "[100]\tvalid_0's cross_entropy: 0.449634\n",
      "[101]\tvalid_0's cross_entropy: 0.449546\n",
      "[102]\tvalid_0's cross_entropy: 0.449483\n",
      "[103]\tvalid_0's cross_entropy: 0.449403\n",
      "[104]\tvalid_0's cross_entropy: 0.449312\n",
      "[105]\tvalid_0's cross_entropy: 0.449233\n",
      "[106]\tvalid_0's cross_entropy: 0.449166\n",
      "[107]\tvalid_0's cross_entropy: 0.449092\n",
      "[108]\tvalid_0's cross_entropy: 0.449017\n",
      "[109]\tvalid_0's cross_entropy: 0.448942\n",
      "[110]\tvalid_0's cross_entropy: 0.448865\n",
      "[111]\tvalid_0's cross_entropy: 0.448795\n",
      "[112]\tvalid_0's cross_entropy: 0.448707\n",
      "[113]\tvalid_0's cross_entropy: 0.44864\n",
      "[114]\tvalid_0's cross_entropy: 0.448572\n",
      "[115]\tvalid_0's cross_entropy: 0.448495\n",
      "[116]\tvalid_0's cross_entropy: 0.448422\n",
      "[117]\tvalid_0's cross_entropy: 0.448359\n",
      "[118]\tvalid_0's cross_entropy: 0.448295\n",
      "[119]\tvalid_0's cross_entropy: 0.448227\n",
      "[120]\tvalid_0's cross_entropy: 0.448158\n",
      "[121]\tvalid_0's cross_entropy: 0.448079\n",
      "[122]\tvalid_0's cross_entropy: 0.448011\n",
      "[123]\tvalid_0's cross_entropy: 0.447934\n",
      "[124]\tvalid_0's cross_entropy: 0.447848\n",
      "[125]\tvalid_0's cross_entropy: 0.447783\n",
      "[126]\tvalid_0's cross_entropy: 0.447727\n",
      "[127]\tvalid_0's cross_entropy: 0.447669\n",
      "[128]\tvalid_0's cross_entropy: 0.447604\n",
      "[129]\tvalid_0's cross_entropy: 0.447548\n",
      "[130]\tvalid_0's cross_entropy: 0.447485\n",
      "[131]\tvalid_0's cross_entropy: 0.447427\n",
      "[132]\tvalid_0's cross_entropy: 0.447359\n",
      "[133]\tvalid_0's cross_entropy: 0.447297\n",
      "[134]\tvalid_0's cross_entropy: 0.447252\n",
      "[135]\tvalid_0's cross_entropy: 0.447179\n",
      "[136]\tvalid_0's cross_entropy: 0.447112\n",
      "[137]\tvalid_0's cross_entropy: 0.447063\n",
      "[138]\tvalid_0's cross_entropy: 0.447011\n",
      "[139]\tvalid_0's cross_entropy: 0.446957\n",
      "[140]\tvalid_0's cross_entropy: 0.446908\n",
      "[141]\tvalid_0's cross_entropy: 0.44687\n",
      "[142]\tvalid_0's cross_entropy: 0.446806\n",
      "[143]\tvalid_0's cross_entropy: 0.446759\n",
      "[144]\tvalid_0's cross_entropy: 0.446703\n",
      "[145]\tvalid_0's cross_entropy: 0.446642\n",
      "[146]\tvalid_0's cross_entropy: 0.446581\n",
      "[147]\tvalid_0's cross_entropy: 0.446536\n",
      "[148]\tvalid_0's cross_entropy: 0.446494\n",
      "[149]\tvalid_0's cross_entropy: 0.44644\n",
      "[150]\tvalid_0's cross_entropy: 0.446393\n",
      "[151]\tvalid_0's cross_entropy: 0.44634\n",
      "[152]\tvalid_0's cross_entropy: 0.446282\n",
      "[153]\tvalid_0's cross_entropy: 0.446229\n",
      "[154]\tvalid_0's cross_entropy: 0.446189\n",
      "[155]\tvalid_0's cross_entropy: 0.446146\n",
      "[156]\tvalid_0's cross_entropy: 0.446101\n",
      "[157]\tvalid_0's cross_entropy: 0.446059\n",
      "[158]\tvalid_0's cross_entropy: 0.446018\n",
      "[159]\tvalid_0's cross_entropy: 0.445981\n",
      "[160]\tvalid_0's cross_entropy: 0.445939\n",
      "[161]\tvalid_0's cross_entropy: 0.445883\n",
      "[162]\tvalid_0's cross_entropy: 0.445842\n",
      "[163]\tvalid_0's cross_entropy: 0.445798\n",
      "[164]\tvalid_0's cross_entropy: 0.445756\n",
      "[165]\tvalid_0's cross_entropy: 0.445704\n",
      "[166]\tvalid_0's cross_entropy: 0.445671\n",
      "[167]\tvalid_0's cross_entropy: 0.445632\n",
      "[168]\tvalid_0's cross_entropy: 0.445602\n",
      "[169]\tvalid_0's cross_entropy: 0.445563\n",
      "[170]\tvalid_0's cross_entropy: 0.445509\n",
      "[171]\tvalid_0's cross_entropy: 0.445465\n",
      "[172]\tvalid_0's cross_entropy: 0.445424\n",
      "[173]\tvalid_0's cross_entropy: 0.445371\n",
      "[174]\tvalid_0's cross_entropy: 0.445335\n",
      "[175]\tvalid_0's cross_entropy: 0.445304\n",
      "[176]\tvalid_0's cross_entropy: 0.44527\n",
      "[177]\tvalid_0's cross_entropy: 0.44522\n",
      "[178]\tvalid_0's cross_entropy: 0.44519\n",
      "[179]\tvalid_0's cross_entropy: 0.445147\n",
      "[180]\tvalid_0's cross_entropy: 0.445112\n",
      "[181]\tvalid_0's cross_entropy: 0.44507\n",
      "[182]\tvalid_0's cross_entropy: 0.445037\n",
      "[183]\tvalid_0's cross_entropy: 0.445003\n",
      "[184]\tvalid_0's cross_entropy: 0.444969\n",
      "[185]\tvalid_0's cross_entropy: 0.444933\n",
      "[186]\tvalid_0's cross_entropy: 0.444893\n",
      "[187]\tvalid_0's cross_entropy: 0.444859\n",
      "[188]\tvalid_0's cross_entropy: 0.444826\n",
      "[189]\tvalid_0's cross_entropy: 0.444793\n",
      "[190]\tvalid_0's cross_entropy: 0.444767\n",
      "[191]\tvalid_0's cross_entropy: 0.444732\n",
      "[192]\tvalid_0's cross_entropy: 0.444701\n",
      "[193]\tvalid_0's cross_entropy: 0.444661\n",
      "[194]\tvalid_0's cross_entropy: 0.444645\n",
      "[195]\tvalid_0's cross_entropy: 0.444607\n",
      "[196]\tvalid_0's cross_entropy: 0.444575\n",
      "[197]\tvalid_0's cross_entropy: 0.444547\n",
      "[198]\tvalid_0's cross_entropy: 0.444519\n",
      "[199]\tvalid_0's cross_entropy: 0.444483\n",
      "[200]\tvalid_0's cross_entropy: 0.444428\n",
      "[201]\tvalid_0's cross_entropy: 0.444385\n",
      "[202]\tvalid_0's cross_entropy: 0.44436\n",
      "[203]\tvalid_0's cross_entropy: 0.444322\n",
      "[204]\tvalid_0's cross_entropy: 0.444279\n",
      "[205]\tvalid_0's cross_entropy: 0.444248\n",
      "[206]\tvalid_0's cross_entropy: 0.444209\n",
      "[207]\tvalid_0's cross_entropy: 0.444169\n",
      "[208]\tvalid_0's cross_entropy: 0.444132\n",
      "[209]\tvalid_0's cross_entropy: 0.444106\n",
      "[210]\tvalid_0's cross_entropy: 0.44408\n",
      "[211]\tvalid_0's cross_entropy: 0.444056\n",
      "[212]\tvalid_0's cross_entropy: 0.444032\n",
      "[213]\tvalid_0's cross_entropy: 0.444011\n",
      "[214]\tvalid_0's cross_entropy: 0.443976\n",
      "[215]\tvalid_0's cross_entropy: 0.443939\n",
      "[216]\tvalid_0's cross_entropy: 0.443912\n",
      "[217]\tvalid_0's cross_entropy: 0.443895\n",
      "[218]\tvalid_0's cross_entropy: 0.443867\n",
      "[219]\tvalid_0's cross_entropy: 0.443832\n",
      "[220]\tvalid_0's cross_entropy: 0.4438\n",
      "[221]\tvalid_0's cross_entropy: 0.443774\n",
      "[222]\tvalid_0's cross_entropy: 0.443743\n",
      "[223]\tvalid_0's cross_entropy: 0.443717\n",
      "[224]\tvalid_0's cross_entropy: 0.443693\n",
      "[225]\tvalid_0's cross_entropy: 0.443666\n",
      "[226]\tvalid_0's cross_entropy: 0.443648\n",
      "[227]\tvalid_0's cross_entropy: 0.443612\n",
      "[228]\tvalid_0's cross_entropy: 0.443578\n",
      "[229]\tvalid_0's cross_entropy: 0.443536\n",
      "[230]\tvalid_0's cross_entropy: 0.443515\n",
      "[231]\tvalid_0's cross_entropy: 0.443498\n",
      "[232]\tvalid_0's cross_entropy: 0.443475\n",
      "[233]\tvalid_0's cross_entropy: 0.443462\n",
      "[234]\tvalid_0's cross_entropy: 0.44343\n",
      "[235]\tvalid_0's cross_entropy: 0.443405\n",
      "[236]\tvalid_0's cross_entropy: 0.443378\n",
      "[237]\tvalid_0's cross_entropy: 0.443356\n",
      "[238]\tvalid_0's cross_entropy: 0.443331\n",
      "[239]\tvalid_0's cross_entropy: 0.443312\n",
      "[240]\tvalid_0's cross_entropy: 0.443287\n",
      "[241]\tvalid_0's cross_entropy: 0.443247\n",
      "[242]\tvalid_0's cross_entropy: 0.44322\n",
      "[243]\tvalid_0's cross_entropy: 0.4432\n",
      "[244]\tvalid_0's cross_entropy: 0.443176\n",
      "[245]\tvalid_0's cross_entropy: 0.443145\n",
      "[246]\tvalid_0's cross_entropy: 0.44313\n",
      "[247]\tvalid_0's cross_entropy: 0.443108\n",
      "[248]\tvalid_0's cross_entropy: 0.443088\n",
      "[249]\tvalid_0's cross_entropy: 0.443058\n",
      "[250]\tvalid_0's cross_entropy: 0.443032\n",
      "[251]\tvalid_0's cross_entropy: 0.443006\n",
      "[252]\tvalid_0's cross_entropy: 0.442984\n",
      "[253]\tvalid_0's cross_entropy: 0.442945\n",
      "[254]\tvalid_0's cross_entropy: 0.44293\n",
      "[255]\tvalid_0's cross_entropy: 0.44291\n",
      "[256]\tvalid_0's cross_entropy: 0.442887\n",
      "[257]\tvalid_0's cross_entropy: 0.442866\n",
      "[258]\tvalid_0's cross_entropy: 0.442844\n",
      "[259]\tvalid_0's cross_entropy: 0.442826\n",
      "[260]\tvalid_0's cross_entropy: 0.442807\n",
      "[261]\tvalid_0's cross_entropy: 0.442794\n",
      "[262]\tvalid_0's cross_entropy: 0.44277\n",
      "[263]\tvalid_0's cross_entropy: 0.442756\n",
      "[264]\tvalid_0's cross_entropy: 0.442742\n",
      "[265]\tvalid_0's cross_entropy: 0.442728\n",
      "[266]\tvalid_0's cross_entropy: 0.44271\n",
      "[267]\tvalid_0's cross_entropy: 0.442687\n",
      "[268]\tvalid_0's cross_entropy: 0.442676\n",
      "[269]\tvalid_0's cross_entropy: 0.442663\n",
      "[270]\tvalid_0's cross_entropy: 0.442625\n",
      "[271]\tvalid_0's cross_entropy: 0.442615\n",
      "[272]\tvalid_0's cross_entropy: 0.442595\n",
      "[273]\tvalid_0's cross_entropy: 0.442565\n",
      "[274]\tvalid_0's cross_entropy: 0.442538\n",
      "[275]\tvalid_0's cross_entropy: 0.442523\n",
      "[276]\tvalid_0's cross_entropy: 0.442503\n",
      "[277]\tvalid_0's cross_entropy: 0.442473\n",
      "[278]\tvalid_0's cross_entropy: 0.442454\n",
      "[279]\tvalid_0's cross_entropy: 0.442426\n",
      "[280]\tvalid_0's cross_entropy: 0.442404\n",
      "[281]\tvalid_0's cross_entropy: 0.442377\n",
      "[282]\tvalid_0's cross_entropy: 0.442364\n",
      "[283]\tvalid_0's cross_entropy: 0.442338\n",
      "[284]\tvalid_0's cross_entropy: 0.442325\n",
      "[285]\tvalid_0's cross_entropy: 0.442298\n",
      "[286]\tvalid_0's cross_entropy: 0.442274\n",
      "[287]\tvalid_0's cross_entropy: 0.442246\n",
      "[288]\tvalid_0's cross_entropy: 0.442222\n",
      "[289]\tvalid_0's cross_entropy: 0.442198\n",
      "[290]\tvalid_0's cross_entropy: 0.442175\n",
      "[291]\tvalid_0's cross_entropy: 0.442161\n",
      "[292]\tvalid_0's cross_entropy: 0.442121\n",
      "[293]\tvalid_0's cross_entropy: 0.442098\n",
      "[294]\tvalid_0's cross_entropy: 0.442084\n",
      "[295]\tvalid_0's cross_entropy: 0.442064\n",
      "[296]\tvalid_0's cross_entropy: 0.442053\n",
      "[297]\tvalid_0's cross_entropy: 0.442039\n",
      "[298]\tvalid_0's cross_entropy: 0.442015\n",
      "[299]\tvalid_0's cross_entropy: 0.442002\n",
      "[300]\tvalid_0's cross_entropy: 0.441978\n",
      "[301]\tvalid_0's cross_entropy: 0.441957\n",
      "[302]\tvalid_0's cross_entropy: 0.441939\n",
      "[303]\tvalid_0's cross_entropy: 0.441932\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[304]\tvalid_0's cross_entropy: 0.441902\n",
      "[305]\tvalid_0's cross_entropy: 0.441884\n",
      "[306]\tvalid_0's cross_entropy: 0.441857\n",
      "[307]\tvalid_0's cross_entropy: 0.441841\n",
      "[308]\tvalid_0's cross_entropy: 0.441833\n",
      "[309]\tvalid_0's cross_entropy: 0.441821\n",
      "[310]\tvalid_0's cross_entropy: 0.441799\n",
      "[311]\tvalid_0's cross_entropy: 0.441777\n",
      "[312]\tvalid_0's cross_entropy: 0.441766\n",
      "[313]\tvalid_0's cross_entropy: 0.441752\n",
      "[314]\tvalid_0's cross_entropy: 0.441727\n",
      "[315]\tvalid_0's cross_entropy: 0.441698\n",
      "[316]\tvalid_0's cross_entropy: 0.441683\n",
      "[317]\tvalid_0's cross_entropy: 0.441676\n",
      "[318]\tvalid_0's cross_entropy: 0.441657\n",
      "[319]\tvalid_0's cross_entropy: 0.441639\n",
      "[320]\tvalid_0's cross_entropy: 0.441619\n",
      "[321]\tvalid_0's cross_entropy: 0.441603\n",
      "[322]\tvalid_0's cross_entropy: 0.441594\n",
      "[323]\tvalid_0's cross_entropy: 0.441581\n",
      "[324]\tvalid_0's cross_entropy: 0.441572\n",
      "[325]\tvalid_0's cross_entropy: 0.441553\n",
      "[326]\tvalid_0's cross_entropy: 0.441537\n",
      "[327]\tvalid_0's cross_entropy: 0.441521\n",
      "[328]\tvalid_0's cross_entropy: 0.441505\n",
      "[329]\tvalid_0's cross_entropy: 0.441493\n",
      "[330]\tvalid_0's cross_entropy: 0.441469\n",
      "[331]\tvalid_0's cross_entropy: 0.441455\n",
      "[332]\tvalid_0's cross_entropy: 0.441434\n",
      "[333]\tvalid_0's cross_entropy: 0.44142\n",
      "[334]\tvalid_0's cross_entropy: 0.441404\n",
      "[335]\tvalid_0's cross_entropy: 0.441388\n",
      "[336]\tvalid_0's cross_entropy: 0.441371\n",
      "[337]\tvalid_0's cross_entropy: 0.441366\n",
      "[338]\tvalid_0's cross_entropy: 0.441357\n",
      "[339]\tvalid_0's cross_entropy: 0.441333\n",
      "[340]\tvalid_0's cross_entropy: 0.441324\n",
      "[341]\tvalid_0's cross_entropy: 0.441309\n",
      "[342]\tvalid_0's cross_entropy: 0.441294\n",
      "[343]\tvalid_0's cross_entropy: 0.441281\n",
      "[344]\tvalid_0's cross_entropy: 0.441249\n",
      "[345]\tvalid_0's cross_entropy: 0.441237\n",
      "[346]\tvalid_0's cross_entropy: 0.441222\n",
      "[347]\tvalid_0's cross_entropy: 0.441215\n",
      "[348]\tvalid_0's cross_entropy: 0.441196\n",
      "[349]\tvalid_0's cross_entropy: 0.441189\n",
      "[350]\tvalid_0's cross_entropy: 0.44118\n",
      "[351]\tvalid_0's cross_entropy: 0.441172\n",
      "[352]\tvalid_0's cross_entropy: 0.441163\n",
      "[353]\tvalid_0's cross_entropy: 0.441156\n",
      "[354]\tvalid_0's cross_entropy: 0.441138\n",
      "[355]\tvalid_0's cross_entropy: 0.441112\n",
      "[356]\tvalid_0's cross_entropy: 0.441108\n",
      "[357]\tvalid_0's cross_entropy: 0.441096\n",
      "[358]\tvalid_0's cross_entropy: 0.44109\n",
      "[359]\tvalid_0's cross_entropy: 0.441075\n",
      "[360]\tvalid_0's cross_entropy: 0.441053\n",
      "[361]\tvalid_0's cross_entropy: 0.44104\n",
      "[362]\tvalid_0's cross_entropy: 0.441022\n",
      "[363]\tvalid_0's cross_entropy: 0.440999\n",
      "[364]\tvalid_0's cross_entropy: 0.440991\n",
      "[365]\tvalid_0's cross_entropy: 0.440979\n",
      "[366]\tvalid_0's cross_entropy: 0.440976\n",
      "[367]\tvalid_0's cross_entropy: 0.440954\n",
      "[368]\tvalid_0's cross_entropy: 0.440941\n",
      "[369]\tvalid_0's cross_entropy: 0.440932\n",
      "[370]\tvalid_0's cross_entropy: 0.440914\n",
      "[371]\tvalid_0's cross_entropy: 0.44091\n",
      "[372]\tvalid_0's cross_entropy: 0.440894\n",
      "[373]\tvalid_0's cross_entropy: 0.440876\n",
      "[374]\tvalid_0's cross_entropy: 0.440858\n",
      "[375]\tvalid_0's cross_entropy: 0.440854\n",
      "[376]\tvalid_0's cross_entropy: 0.440837\n",
      "[377]\tvalid_0's cross_entropy: 0.440833\n",
      "[378]\tvalid_0's cross_entropy: 0.440817\n",
      "[379]\tvalid_0's cross_entropy: 0.440815\n",
      "[380]\tvalid_0's cross_entropy: 0.440821\n",
      "[381]\tvalid_0's cross_entropy: 0.440806\n",
      "[382]\tvalid_0's cross_entropy: 0.440791\n",
      "[383]\tvalid_0's cross_entropy: 0.440784\n",
      "[384]\tvalid_0's cross_entropy: 0.440779\n",
      "[385]\tvalid_0's cross_entropy: 0.440775\n",
      "[386]\tvalid_0's cross_entropy: 0.440771\n",
      "[387]\tvalid_0's cross_entropy: 0.440763\n",
      "[388]\tvalid_0's cross_entropy: 0.440742\n",
      "[389]\tvalid_0's cross_entropy: 0.440735\n",
      "[390]\tvalid_0's cross_entropy: 0.440726\n",
      "[391]\tvalid_0's cross_entropy: 0.440709\n",
      "[392]\tvalid_0's cross_entropy: 0.440706\n",
      "[393]\tvalid_0's cross_entropy: 0.440695\n",
      "[394]\tvalid_0's cross_entropy: 0.440673\n",
      "[395]\tvalid_0's cross_entropy: 0.440662\n",
      "[396]\tvalid_0's cross_entropy: 0.440653\n",
      "[397]\tvalid_0's cross_entropy: 0.440648\n",
      "[398]\tvalid_0's cross_entropy: 0.440652\n",
      "[399]\tvalid_0's cross_entropy: 0.440645\n",
      "[400]\tvalid_0's cross_entropy: 0.440636\n",
      "[401]\tvalid_0's cross_entropy: 0.440628\n",
      "[402]\tvalid_0's cross_entropy: 0.440618\n",
      "[403]\tvalid_0's cross_entropy: 0.440598\n",
      "[404]\tvalid_0's cross_entropy: 0.440576\n",
      "[405]\tvalid_0's cross_entropy: 0.440563\n",
      "[406]\tvalid_0's cross_entropy: 0.44054\n",
      "[407]\tvalid_0's cross_entropy: 0.440538\n",
      "[408]\tvalid_0's cross_entropy: 0.44053\n",
      "[409]\tvalid_0's cross_entropy: 0.440511\n",
      "[410]\tvalid_0's cross_entropy: 0.440506\n",
      "[411]\tvalid_0's cross_entropy: 0.440499\n",
      "[412]\tvalid_0's cross_entropy: 0.440497\n",
      "[413]\tvalid_0's cross_entropy: 0.440495\n",
      "[414]\tvalid_0's cross_entropy: 0.440491\n",
      "[415]\tvalid_0's cross_entropy: 0.44049\n",
      "[416]\tvalid_0's cross_entropy: 0.44049\n",
      "[417]\tvalid_0's cross_entropy: 0.44048\n",
      "[418]\tvalid_0's cross_entropy: 0.440462\n",
      "[419]\tvalid_0's cross_entropy: 0.440457\n",
      "[420]\tvalid_0's cross_entropy: 0.440452\n",
      "[421]\tvalid_0's cross_entropy: 0.440431\n",
      "[422]\tvalid_0's cross_entropy: 0.44041\n",
      "[423]\tvalid_0's cross_entropy: 0.440406\n",
      "[424]\tvalid_0's cross_entropy: 0.440391\n",
      "[425]\tvalid_0's cross_entropy: 0.44038\n",
      "[426]\tvalid_0's cross_entropy: 0.440379\n",
      "[427]\tvalid_0's cross_entropy: 0.440354\n",
      "[428]\tvalid_0's cross_entropy: 0.44034\n",
      "[429]\tvalid_0's cross_entropy: 0.440346\n",
      "[430]\tvalid_0's cross_entropy: 0.440333\n",
      "[431]\tvalid_0's cross_entropy: 0.440321\n",
      "[432]\tvalid_0's cross_entropy: 0.440302\n",
      "[433]\tvalid_0's cross_entropy: 0.440296\n",
      "[434]\tvalid_0's cross_entropy: 0.440285\n",
      "[435]\tvalid_0's cross_entropy: 0.440282\n",
      "[436]\tvalid_0's cross_entropy: 0.440265\n",
      "[437]\tvalid_0's cross_entropy: 0.440258\n",
      "[438]\tvalid_0's cross_entropy: 0.44025\n",
      "[439]\tvalid_0's cross_entropy: 0.440246\n",
      "[440]\tvalid_0's cross_entropy: 0.440229\n",
      "[441]\tvalid_0's cross_entropy: 0.440231\n",
      "[442]\tvalid_0's cross_entropy: 0.440235\n",
      "[443]\tvalid_0's cross_entropy: 0.44022\n",
      "[444]\tvalid_0's cross_entropy: 0.440214\n",
      "[445]\tvalid_0's cross_entropy: 0.44021\n",
      "[446]\tvalid_0's cross_entropy: 0.440202\n",
      "[447]\tvalid_0's cross_entropy: 0.440206\n",
      "[448]\tvalid_0's cross_entropy: 0.440201\n",
      "[449]\tvalid_0's cross_entropy: 0.440193\n",
      "[450]\tvalid_0's cross_entropy: 0.440179\n",
      "[451]\tvalid_0's cross_entropy: 0.440176\n",
      "[452]\tvalid_0's cross_entropy: 0.440165\n",
      "[453]\tvalid_0's cross_entropy: 0.440161\n",
      "[454]\tvalid_0's cross_entropy: 0.440154\n",
      "[455]\tvalid_0's cross_entropy: 0.440155\n",
      "[456]\tvalid_0's cross_entropy: 0.440149\n",
      "[457]\tvalid_0's cross_entropy: 0.440146\n",
      "[458]\tvalid_0's cross_entropy: 0.440135\n",
      "[459]\tvalid_0's cross_entropy: 0.44014\n",
      "[460]\tvalid_0's cross_entropy: 0.440129\n",
      "[461]\tvalid_0's cross_entropy: 0.440122\n",
      "[462]\tvalid_0's cross_entropy: 0.440117\n",
      "[463]\tvalid_0's cross_entropy: 0.440113\n",
      "[464]\tvalid_0's cross_entropy: 0.440114\n",
      "[465]\tvalid_0's cross_entropy: 0.440101\n",
      "[466]\tvalid_0's cross_entropy: 0.440101\n",
      "[467]\tvalid_0's cross_entropy: 0.440085\n",
      "[468]\tvalid_0's cross_entropy: 0.440087\n",
      "[469]\tvalid_0's cross_entropy: 0.440078\n",
      "[470]\tvalid_0's cross_entropy: 0.440077\n",
      "[471]\tvalid_0's cross_entropy: 0.440078\n",
      "[472]\tvalid_0's cross_entropy: 0.440067\n",
      "[473]\tvalid_0's cross_entropy: 0.440056\n",
      "[474]\tvalid_0's cross_entropy: 0.44005\n",
      "[475]\tvalid_0's cross_entropy: 0.440042\n",
      "[476]\tvalid_0's cross_entropy: 0.440041\n",
      "[477]\tvalid_0's cross_entropy: 0.440044\n",
      "[478]\tvalid_0's cross_entropy: 0.440045\n",
      "[479]\tvalid_0's cross_entropy: 0.44003\n",
      "[480]\tvalid_0's cross_entropy: 0.440021\n",
      "[481]\tvalid_0's cross_entropy: 0.440032\n",
      "[482]\tvalid_0's cross_entropy: 0.44002\n",
      "[483]\tvalid_0's cross_entropy: 0.440014\n",
      "[484]\tvalid_0's cross_entropy: 0.440005\n",
      "[485]\tvalid_0's cross_entropy: 0.439992\n",
      "[486]\tvalid_0's cross_entropy: 0.439986\n",
      "[487]\tvalid_0's cross_entropy: 0.439973\n",
      "[488]\tvalid_0's cross_entropy: 0.439968\n",
      "[489]\tvalid_0's cross_entropy: 0.439966\n",
      "[490]\tvalid_0's cross_entropy: 0.439965\n",
      "[491]\tvalid_0's cross_entropy: 0.439968\n",
      "[492]\tvalid_0's cross_entropy: 0.439962\n",
      "[493]\tvalid_0's cross_entropy: 0.439959\n",
      "[494]\tvalid_0's cross_entropy: 0.439963\n",
      "[495]\tvalid_0's cross_entropy: 0.439943\n",
      "[496]\tvalid_0's cross_entropy: 0.439937\n",
      "[497]\tvalid_0's cross_entropy: 0.439933\n",
      "[498]\tvalid_0's cross_entropy: 0.439932\n",
      "[499]\tvalid_0's cross_entropy: 0.439921\n",
      "[500]\tvalid_0's cross_entropy: 0.439921\n",
      "[501]\tvalid_0's cross_entropy: 0.439902\n",
      "[502]\tvalid_0's cross_entropy: 0.439904\n",
      "[503]\tvalid_0's cross_entropy: 0.439893\n",
      "[504]\tvalid_0's cross_entropy: 0.43988\n",
      "[505]\tvalid_0's cross_entropy: 0.439873\n",
      "[506]\tvalid_0's cross_entropy: 0.439876\n",
      "[507]\tvalid_0's cross_entropy: 0.439872\n",
      "[508]\tvalid_0's cross_entropy: 0.439867\n",
      "[509]\tvalid_0's cross_entropy: 0.439867\n",
      "[510]\tvalid_0's cross_entropy: 0.439846\n",
      "[511]\tvalid_0's cross_entropy: 0.439835\n",
      "[512]\tvalid_0's cross_entropy: 0.439827\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[513]\tvalid_0's cross_entropy: 0.439819\n",
      "[514]\tvalid_0's cross_entropy: 0.439818\n",
      "[515]\tvalid_0's cross_entropy: 0.439812\n",
      "[516]\tvalid_0's cross_entropy: 0.439815\n",
      "[517]\tvalid_0's cross_entropy: 0.439817\n",
      "[518]\tvalid_0's cross_entropy: 0.439812\n",
      "[519]\tvalid_0's cross_entropy: 0.439804\n",
      "[520]\tvalid_0's cross_entropy: 0.439802\n",
      "[521]\tvalid_0's cross_entropy: 0.439795\n",
      "[522]\tvalid_0's cross_entropy: 0.43979\n",
      "[523]\tvalid_0's cross_entropy: 0.439789\n",
      "[524]\tvalid_0's cross_entropy: 0.439788\n",
      "[525]\tvalid_0's cross_entropy: 0.439781\n",
      "[526]\tvalid_0's cross_entropy: 0.43978\n",
      "[527]\tvalid_0's cross_entropy: 0.439776\n",
      "[528]\tvalid_0's cross_entropy: 0.439769\n",
      "[529]\tvalid_0's cross_entropy: 0.439766\n",
      "[530]\tvalid_0's cross_entropy: 0.439758\n",
      "[531]\tvalid_0's cross_entropy: 0.439746\n",
      "[532]\tvalid_0's cross_entropy: 0.439731\n",
      "[533]\tvalid_0's cross_entropy: 0.439726\n",
      "[534]\tvalid_0's cross_entropy: 0.439727\n",
      "[535]\tvalid_0's cross_entropy: 0.439727\n",
      "[536]\tvalid_0's cross_entropy: 0.439725\n",
      "[537]\tvalid_0's cross_entropy: 0.439731\n",
      "[538]\tvalid_0's cross_entropy: 0.439723\n",
      "[539]\tvalid_0's cross_entropy: 0.43972\n",
      "[540]\tvalid_0's cross_entropy: 0.439712\n",
      "[541]\tvalid_0's cross_entropy: 0.43972\n",
      "[542]\tvalid_0's cross_entropy: 0.439711\n",
      "[543]\tvalid_0's cross_entropy: 0.43971\n",
      "[544]\tvalid_0's cross_entropy: 0.439707\n",
      "[545]\tvalid_0's cross_entropy: 0.439704\n",
      "[546]\tvalid_0's cross_entropy: 0.439703\n",
      "[547]\tvalid_0's cross_entropy: 0.439699\n",
      "[548]\tvalid_0's cross_entropy: 0.439679\n",
      "[549]\tvalid_0's cross_entropy: 0.439675\n",
      "[550]\tvalid_0's cross_entropy: 0.439673\n",
      "[551]\tvalid_0's cross_entropy: 0.439667\n",
      "[552]\tvalid_0's cross_entropy: 0.439666\n",
      "[553]\tvalid_0's cross_entropy: 0.439668\n",
      "[554]\tvalid_0's cross_entropy: 0.439669\n",
      "[555]\tvalid_0's cross_entropy: 0.43967\n",
      "[556]\tvalid_0's cross_entropy: 0.43966\n",
      "[557]\tvalid_0's cross_entropy: 0.439648\n",
      "[558]\tvalid_0's cross_entropy: 0.43964\n",
      "[559]\tvalid_0's cross_entropy: 0.439634\n",
      "[560]\tvalid_0's cross_entropy: 0.439633\n",
      "[561]\tvalid_0's cross_entropy: 0.439631\n",
      "[562]\tvalid_0's cross_entropy: 0.439629\n",
      "[563]\tvalid_0's cross_entropy: 0.439626\n",
      "[564]\tvalid_0's cross_entropy: 0.439621\n",
      "[565]\tvalid_0's cross_entropy: 0.439616\n",
      "[566]\tvalid_0's cross_entropy: 0.439611\n",
      "[567]\tvalid_0's cross_entropy: 0.439606\n",
      "[568]\tvalid_0's cross_entropy: 0.439608\n",
      "[569]\tvalid_0's cross_entropy: 0.439607\n",
      "[570]\tvalid_0's cross_entropy: 0.439596\n",
      "[571]\tvalid_0's cross_entropy: 0.439592\n",
      "[572]\tvalid_0's cross_entropy: 0.43959\n",
      "[573]\tvalid_0's cross_entropy: 0.439595\n",
      "[574]\tvalid_0's cross_entropy: 0.439582\n",
      "[575]\tvalid_0's cross_entropy: 0.439578\n",
      "[576]\tvalid_0's cross_entropy: 0.439584\n",
      "[577]\tvalid_0's cross_entropy: 0.439573\n",
      "[578]\tvalid_0's cross_entropy: 0.439557\n",
      "[579]\tvalid_0's cross_entropy: 0.439559\n",
      "[580]\tvalid_0's cross_entropy: 0.439554\n",
      "[581]\tvalid_0's cross_entropy: 0.43955\n",
      "[582]\tvalid_0's cross_entropy: 0.43955\n",
      "[583]\tvalid_0's cross_entropy: 0.439549\n",
      "[584]\tvalid_0's cross_entropy: 0.439546\n",
      "[585]\tvalid_0's cross_entropy: 0.439541\n",
      "[586]\tvalid_0's cross_entropy: 0.439535\n",
      "[587]\tvalid_0's cross_entropy: 0.439533\n",
      "[588]\tvalid_0's cross_entropy: 0.439533\n",
      "[589]\tvalid_0's cross_entropy: 0.439533\n",
      "[590]\tvalid_0's cross_entropy: 0.439537\n",
      "[591]\tvalid_0's cross_entropy: 0.439541\n",
      "[592]\tvalid_0's cross_entropy: 0.439539\n",
      "[593]\tvalid_0's cross_entropy: 0.439533\n",
      "[594]\tvalid_0's cross_entropy: 0.439536\n",
      "[595]\tvalid_0's cross_entropy: 0.439528\n",
      "[596]\tvalid_0's cross_entropy: 0.439522\n",
      "[597]\tvalid_0's cross_entropy: 0.439526\n",
      "[598]\tvalid_0's cross_entropy: 0.439533\n",
      "[599]\tvalid_0's cross_entropy: 0.439536\n",
      "[600]\tvalid_0's cross_entropy: 0.439533\n",
      "[601]\tvalid_0's cross_entropy: 0.439526\n",
      "[602]\tvalid_0's cross_entropy: 0.439532\n",
      "[603]\tvalid_0's cross_entropy: 0.439529\n",
      "[604]\tvalid_0's cross_entropy: 0.439529\n",
      "[605]\tvalid_0's cross_entropy: 0.439531\n",
      "[606]\tvalid_0's cross_entropy: 0.439529\n",
      "[607]\tvalid_0's cross_entropy: 0.439524\n",
      "[608]\tvalid_0's cross_entropy: 0.439531\n",
      "[609]\tvalid_0's cross_entropy: 0.439525\n",
      "[610]\tvalid_0's cross_entropy: 0.439519\n",
      "[611]\tvalid_0's cross_entropy: 0.439525\n",
      "[612]\tvalid_0's cross_entropy: 0.439524\n",
      "[613]\tvalid_0's cross_entropy: 0.439535\n",
      "[614]\tvalid_0's cross_entropy: 0.439526\n",
      "[615]\tvalid_0's cross_entropy: 0.439525\n",
      "[616]\tvalid_0's cross_entropy: 0.439524\n",
      "[617]\tvalid_0's cross_entropy: 0.439522\n",
      "[618]\tvalid_0's cross_entropy: 0.439528\n",
      "[619]\tvalid_0's cross_entropy: 0.439535\n",
      "[620]\tvalid_0's cross_entropy: 0.439534\n",
      "[621]\tvalid_0's cross_entropy: 0.439533\n",
      "[622]\tvalid_0's cross_entropy: 0.439529\n",
      "[623]\tvalid_0's cross_entropy: 0.439536\n",
      "[624]\tvalid_0's cross_entropy: 0.439538\n",
      "[625]\tvalid_0's cross_entropy: 0.439538\n",
      "[626]\tvalid_0's cross_entropy: 0.439538\n",
      "[627]\tvalid_0's cross_entropy: 0.439536\n",
      "[628]\tvalid_0's cross_entropy: 0.43953\n",
      "[629]\tvalid_0's cross_entropy: 0.439537\n",
      "[630]\tvalid_0's cross_entropy: 0.439534\n",
      "[631]\tvalid_0's cross_entropy: 0.439538\n",
      "[632]\tvalid_0's cross_entropy: 0.439533\n",
      "[633]\tvalid_0's cross_entropy: 0.439535\n",
      "[634]\tvalid_0's cross_entropy: 0.439543\n",
      "[635]\tvalid_0's cross_entropy: 0.439539\n",
      "[636]\tvalid_0's cross_entropy: 0.439543\n",
      "[637]\tvalid_0's cross_entropy: 0.439548\n",
      "[638]\tvalid_0's cross_entropy: 0.439543\n",
      "[639]\tvalid_0's cross_entropy: 0.439538\n",
      "[640]\tvalid_0's cross_entropy: 0.439538\n",
      "[641]\tvalid_0's cross_entropy: 0.439532\n",
      "[642]\tvalid_0's cross_entropy: 0.439529\n",
      "[643]\tvalid_0's cross_entropy: 0.439534\n",
      "[644]\tvalid_0's cross_entropy: 0.43954\n",
      "[645]\tvalid_0's cross_entropy: 0.43954\n",
      "[646]\tvalid_0's cross_entropy: 0.439539\n",
      "[647]\tvalid_0's cross_entropy: 0.439535\n",
      "[648]\tvalid_0's cross_entropy: 0.439536\n",
      "[649]\tvalid_0's cross_entropy: 0.43953\n",
      "[650]\tvalid_0's cross_entropy: 0.439527\n",
      "[651]\tvalid_0's cross_entropy: 0.439516\n",
      "[652]\tvalid_0's cross_entropy: 0.439505\n",
      "[653]\tvalid_0's cross_entropy: 0.439507\n",
      "[654]\tvalid_0's cross_entropy: 0.439511\n",
      "[655]\tvalid_0's cross_entropy: 0.439509\n",
      "[656]\tvalid_0's cross_entropy: 0.439505\n",
      "[657]\tvalid_0's cross_entropy: 0.439509\n",
      "[658]\tvalid_0's cross_entropy: 0.439509\n",
      "[659]\tvalid_0's cross_entropy: 0.439502\n",
      "[660]\tvalid_0's cross_entropy: 0.439503\n",
      "[661]\tvalid_0's cross_entropy: 0.4395\n",
      "[662]\tvalid_0's cross_entropy: 0.439508\n",
      "[663]\tvalid_0's cross_entropy: 0.439507\n",
      "[664]\tvalid_0's cross_entropy: 0.439506\n",
      "[665]\tvalid_0's cross_entropy: 0.439508\n",
      "[666]\tvalid_0's cross_entropy: 0.439507\n",
      "[667]\tvalid_0's cross_entropy: 0.439495\n",
      "[668]\tvalid_0's cross_entropy: 0.439503\n",
      "[669]\tvalid_0's cross_entropy: 0.439504\n",
      "[670]\tvalid_0's cross_entropy: 0.439508\n",
      "[671]\tvalid_0's cross_entropy: 0.439504\n",
      "[672]\tvalid_0's cross_entropy: 0.439502\n",
      "[673]\tvalid_0's cross_entropy: 0.4395\n",
      "[674]\tvalid_0's cross_entropy: 0.439505\n",
      "[675]\tvalid_0's cross_entropy: 0.439508\n",
      "[676]\tvalid_0's cross_entropy: 0.439513\n",
      "[677]\tvalid_0's cross_entropy: 0.439511\n",
      "[678]\tvalid_0's cross_entropy: 0.439512\n",
      "[679]\tvalid_0's cross_entropy: 0.439509\n",
      "[680]\tvalid_0's cross_entropy: 0.439514\n",
      "[681]\tvalid_0's cross_entropy: 0.439515\n",
      "[682]\tvalid_0's cross_entropy: 0.439507\n",
      "[683]\tvalid_0's cross_entropy: 0.439504\n",
      "[684]\tvalid_0's cross_entropy: 0.439503\n",
      "[685]\tvalid_0's cross_entropy: 0.4395\n",
      "[686]\tvalid_0's cross_entropy: 0.439494\n",
      "[687]\tvalid_0's cross_entropy: 0.439489\n",
      "[688]\tvalid_0's cross_entropy: 0.439487\n",
      "[689]\tvalid_0's cross_entropy: 0.439489\n",
      "[690]\tvalid_0's cross_entropy: 0.439488\n",
      "[691]\tvalid_0's cross_entropy: 0.439474\n",
      "[692]\tvalid_0's cross_entropy: 0.439475\n",
      "[693]\tvalid_0's cross_entropy: 0.439478\n",
      "[694]\tvalid_0's cross_entropy: 0.439477\n",
      "[695]\tvalid_0's cross_entropy: 0.439479\n",
      "[696]\tvalid_0's cross_entropy: 0.439479\n",
      "[697]\tvalid_0's cross_entropy: 0.439485\n",
      "[698]\tvalid_0's cross_entropy: 0.43949\n",
      "[699]\tvalid_0's cross_entropy: 0.439487\n",
      "[700]\tvalid_0's cross_entropy: 0.439485\n",
      "[701]\tvalid_0's cross_entropy: 0.439478\n",
      "[702]\tvalid_0's cross_entropy: 0.439473\n",
      "[703]\tvalid_0's cross_entropy: 0.439464\n",
      "[704]\tvalid_0's cross_entropy: 0.439462\n",
      "[705]\tvalid_0's cross_entropy: 0.439463\n",
      "[706]\tvalid_0's cross_entropy: 0.439468\n",
      "[707]\tvalid_0's cross_entropy: 0.43946\n",
      "[708]\tvalid_0's cross_entropy: 0.439454\n",
      "[709]\tvalid_0's cross_entropy: 0.439455\n",
      "[710]\tvalid_0's cross_entropy: 0.43944\n",
      "[711]\tvalid_0's cross_entropy: 0.439445\n",
      "[712]\tvalid_0's cross_entropy: 0.439444\n",
      "[713]\tvalid_0's cross_entropy: 0.439448\n",
      "[714]\tvalid_0's cross_entropy: 0.439458\n",
      "[715]\tvalid_0's cross_entropy: 0.439456\n",
      "[716]\tvalid_0's cross_entropy: 0.439456\n",
      "[717]\tvalid_0's cross_entropy: 0.439448\n",
      "[718]\tvalid_0's cross_entropy: 0.439442\n",
      "[719]\tvalid_0's cross_entropy: 0.439442\n",
      "[720]\tvalid_0's cross_entropy: 0.439444\n",
      "[721]\tvalid_0's cross_entropy: 0.439443\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[722]\tvalid_0's cross_entropy: 0.43944\n",
      "[723]\tvalid_0's cross_entropy: 0.439442\n",
      "[724]\tvalid_0's cross_entropy: 0.439448\n",
      "[725]\tvalid_0's cross_entropy: 0.439448\n",
      "[726]\tvalid_0's cross_entropy: 0.439454\n",
      "[727]\tvalid_0's cross_entropy: 0.439452\n",
      "[728]\tvalid_0's cross_entropy: 0.439453\n",
      "[729]\tvalid_0's cross_entropy: 0.439439\n",
      "[730]\tvalid_0's cross_entropy: 0.43944\n",
      "[731]\tvalid_0's cross_entropy: 0.439441\n",
      "[732]\tvalid_0's cross_entropy: 0.439439\n",
      "[733]\tvalid_0's cross_entropy: 0.439444\n",
      "[734]\tvalid_0's cross_entropy: 0.439449\n",
      "[735]\tvalid_0's cross_entropy: 0.439445\n",
      "[736]\tvalid_0's cross_entropy: 0.439439\n",
      "[737]\tvalid_0's cross_entropy: 0.439437\n",
      "[738]\tvalid_0's cross_entropy: 0.439443\n",
      "[739]\tvalid_0's cross_entropy: 0.439445\n",
      "[740]\tvalid_0's cross_entropy: 0.439447\n",
      "[741]\tvalid_0's cross_entropy: 0.439436\n",
      "[742]\tvalid_0's cross_entropy: 0.439433\n",
      "[743]\tvalid_0's cross_entropy: 0.439437\n",
      "[744]\tvalid_0's cross_entropy: 0.43944\n",
      "[745]\tvalid_0's cross_entropy: 0.439438\n",
      "[746]\tvalid_0's cross_entropy: 0.439437\n",
      "[747]\tvalid_0's cross_entropy: 0.439446\n",
      "[748]\tvalid_0's cross_entropy: 0.439437\n",
      "[749]\tvalid_0's cross_entropy: 0.439438\n",
      "[750]\tvalid_0's cross_entropy: 0.439436\n",
      "[751]\tvalid_0's cross_entropy: 0.439437\n",
      "[752]\tvalid_0's cross_entropy: 0.439439\n",
      "[753]\tvalid_0's cross_entropy: 0.439435\n",
      "[754]\tvalid_0's cross_entropy: 0.439427\n",
      "[755]\tvalid_0's cross_entropy: 0.439424\n",
      "[756]\tvalid_0's cross_entropy: 0.439426\n",
      "[757]\tvalid_0's cross_entropy: 0.439429\n",
      "[758]\tvalid_0's cross_entropy: 0.439428\n",
      "[759]\tvalid_0's cross_entropy: 0.439416\n",
      "[760]\tvalid_0's cross_entropy: 0.439416\n",
      "[761]\tvalid_0's cross_entropy: 0.439416\n",
      "[762]\tvalid_0's cross_entropy: 0.439422\n",
      "[763]\tvalid_0's cross_entropy: 0.439424\n",
      "[764]\tvalid_0's cross_entropy: 0.439424\n",
      "[765]\tvalid_0's cross_entropy: 0.439428\n",
      "[766]\tvalid_0's cross_entropy: 0.439423\n",
      "[767]\tvalid_0's cross_entropy: 0.439417\n",
      "[768]\tvalid_0's cross_entropy: 0.439408\n",
      "[769]\tvalid_0's cross_entropy: 0.439409\n",
      "[770]\tvalid_0's cross_entropy: 0.43941\n",
      "[771]\tvalid_0's cross_entropy: 0.439408\n",
      "[772]\tvalid_0's cross_entropy: 0.439409\n",
      "[773]\tvalid_0's cross_entropy: 0.439409\n",
      "[774]\tvalid_0's cross_entropy: 0.439402\n",
      "[775]\tvalid_0's cross_entropy: 0.439403\n",
      "[776]\tvalid_0's cross_entropy: 0.439404\n",
      "[777]\tvalid_0's cross_entropy: 0.439397\n",
      "[778]\tvalid_0's cross_entropy: 0.4394\n",
      "[779]\tvalid_0's cross_entropy: 0.439399\n",
      "[780]\tvalid_0's cross_entropy: 0.439395\n",
      "[781]\tvalid_0's cross_entropy: 0.439392\n",
      "[782]\tvalid_0's cross_entropy: 0.439396\n",
      "[783]\tvalid_0's cross_entropy: 0.439404\n",
      "[784]\tvalid_0's cross_entropy: 0.439401\n",
      "[785]\tvalid_0's cross_entropy: 0.439399\n",
      "[786]\tvalid_0's cross_entropy: 0.439403\n",
      "[787]\tvalid_0's cross_entropy: 0.439392\n",
      "[788]\tvalid_0's cross_entropy: 0.439391\n",
      "[789]\tvalid_0's cross_entropy: 0.439386\n",
      "[790]\tvalid_0's cross_entropy: 0.439389\n",
      "[791]\tvalid_0's cross_entropy: 0.43939\n",
      "[792]\tvalid_0's cross_entropy: 0.439397\n",
      "[793]\tvalid_0's cross_entropy: 0.439398\n",
      "[794]\tvalid_0's cross_entropy: 0.4394\n",
      "[795]\tvalid_0's cross_entropy: 0.439405\n",
      "[796]\tvalid_0's cross_entropy: 0.439405\n",
      "[797]\tvalid_0's cross_entropy: 0.439412\n",
      "[798]\tvalid_0's cross_entropy: 0.439415\n",
      "[799]\tvalid_0's cross_entropy: 0.439411\n",
      "[800]\tvalid_0's cross_entropy: 0.439419\n",
      "[801]\tvalid_0's cross_entropy: 0.439415\n",
      "[802]\tvalid_0's cross_entropy: 0.439407\n",
      "[803]\tvalid_0's cross_entropy: 0.439404\n",
      "[804]\tvalid_0's cross_entropy: 0.439401\n",
      "[805]\tvalid_0's cross_entropy: 0.439398\n",
      "[806]\tvalid_0's cross_entropy: 0.439402\n",
      "[807]\tvalid_0's cross_entropy: 0.439402\n",
      "[808]\tvalid_0's cross_entropy: 0.439404\n",
      "[809]\tvalid_0's cross_entropy: 0.43941\n",
      "[810]\tvalid_0's cross_entropy: 0.43941\n",
      "[811]\tvalid_0's cross_entropy: 0.439407\n",
      "[812]\tvalid_0's cross_entropy: 0.4394\n",
      "[813]\tvalid_0's cross_entropy: 0.439401\n",
      "[814]\tvalid_0's cross_entropy: 0.439402\n",
      "[815]\tvalid_0's cross_entropy: 0.439399\n",
      "[816]\tvalid_0's cross_entropy: 0.4394\n",
      "[817]\tvalid_0's cross_entropy: 0.439405\n",
      "[818]\tvalid_0's cross_entropy: 0.439409\n",
      "[819]\tvalid_0's cross_entropy: 0.439402\n",
      "[820]\tvalid_0's cross_entropy: 0.439405\n",
      "[821]\tvalid_0's cross_entropy: 0.439402\n",
      "[822]\tvalid_0's cross_entropy: 0.439399\n",
      "[823]\tvalid_0's cross_entropy: 0.439394\n",
      "[824]\tvalid_0's cross_entropy: 0.439399\n",
      "[825]\tvalid_0's cross_entropy: 0.439401\n",
      "[826]\tvalid_0's cross_entropy: 0.439401\n",
      "[827]\tvalid_0's cross_entropy: 0.439408\n",
      "[828]\tvalid_0's cross_entropy: 0.439407\n",
      "[829]\tvalid_0's cross_entropy: 0.439401\n",
      "[830]\tvalid_0's cross_entropy: 0.439399\n",
      "[831]\tvalid_0's cross_entropy: 0.439398\n",
      "[832]\tvalid_0's cross_entropy: 0.439392\n",
      "[833]\tvalid_0's cross_entropy: 0.439392\n",
      "[834]\tvalid_0's cross_entropy: 0.439391\n",
      "[835]\tvalid_0's cross_entropy: 0.439388\n",
      "[836]\tvalid_0's cross_entropy: 0.439387\n",
      "[837]\tvalid_0's cross_entropy: 0.439388\n",
      "[838]\tvalid_0's cross_entropy: 0.439389\n",
      "[839]\tvalid_0's cross_entropy: 0.439397\n",
      "[840]\tvalid_0's cross_entropy: 0.439398\n",
      "[841]\tvalid_0's cross_entropy: 0.4394\n",
      "[842]\tvalid_0's cross_entropy: 0.439398\n",
      "[843]\tvalid_0's cross_entropy: 0.439405\n",
      "[844]\tvalid_0's cross_entropy: 0.439399\n",
      "[845]\tvalid_0's cross_entropy: 0.439399\n",
      "[846]\tvalid_0's cross_entropy: 0.439392\n",
      "[847]\tvalid_0's cross_entropy: 0.439395\n",
      "[848]\tvalid_0's cross_entropy: 0.439382\n",
      "[849]\tvalid_0's cross_entropy: 0.43939\n",
      "[850]\tvalid_0's cross_entropy: 0.439389\n",
      "[851]\tvalid_0's cross_entropy: 0.439388\n",
      "[852]\tvalid_0's cross_entropy: 0.439389\n",
      "[853]\tvalid_0's cross_entropy: 0.439391\n",
      "[854]\tvalid_0's cross_entropy: 0.439392\n",
      "[855]\tvalid_0's cross_entropy: 0.439386\n",
      "[856]\tvalid_0's cross_entropy: 0.439389\n",
      "[857]\tvalid_0's cross_entropy: 0.439388\n",
      "[858]\tvalid_0's cross_entropy: 0.439388\n",
      "[859]\tvalid_0's cross_entropy: 0.439386\n",
      "[860]\tvalid_0's cross_entropy: 0.439379\n",
      "[861]\tvalid_0's cross_entropy: 0.439379\n",
      "[862]\tvalid_0's cross_entropy: 0.439382\n",
      "[863]\tvalid_0's cross_entropy: 0.439389\n",
      "[864]\tvalid_0's cross_entropy: 0.439378\n",
      "[865]\tvalid_0's cross_entropy: 0.43938\n",
      "[866]\tvalid_0's cross_entropy: 0.439378\n",
      "[867]\tvalid_0's cross_entropy: 0.439381\n",
      "[868]\tvalid_0's cross_entropy: 0.439381\n",
      "[869]\tvalid_0's cross_entropy: 0.439387\n",
      "[870]\tvalid_0's cross_entropy: 0.439387\n",
      "[871]\tvalid_0's cross_entropy: 0.439392\n",
      "[872]\tvalid_0's cross_entropy: 0.439396\n",
      "[873]\tvalid_0's cross_entropy: 0.439398\n",
      "[874]\tvalid_0's cross_entropy: 0.439398\n",
      "[875]\tvalid_0's cross_entropy: 0.439397\n",
      "[876]\tvalid_0's cross_entropy: 0.439402\n",
      "[877]\tvalid_0's cross_entropy: 0.439407\n",
      "[878]\tvalid_0's cross_entropy: 0.439408\n",
      "[879]\tvalid_0's cross_entropy: 0.43941\n",
      "[880]\tvalid_0's cross_entropy: 0.439405\n",
      "[881]\tvalid_0's cross_entropy: 0.439396\n",
      "[882]\tvalid_0's cross_entropy: 0.439388\n",
      "[883]\tvalid_0's cross_entropy: 0.439387\n",
      "[884]\tvalid_0's cross_entropy: 0.439384\n",
      "[885]\tvalid_0's cross_entropy: 0.439382\n",
      "[886]\tvalid_0's cross_entropy: 0.439388\n",
      "[887]\tvalid_0's cross_entropy: 0.439381\n",
      "[888]\tvalid_0's cross_entropy: 0.439376\n",
      "[889]\tvalid_0's cross_entropy: 0.439381\n",
      "[890]\tvalid_0's cross_entropy: 0.43938\n",
      "[891]\tvalid_0's cross_entropy: 0.439378\n",
      "[892]\tvalid_0's cross_entropy: 0.439383\n",
      "[893]\tvalid_0's cross_entropy: 0.439388\n",
      "[894]\tvalid_0's cross_entropy: 0.439382\n",
      "[895]\tvalid_0's cross_entropy: 0.43938\n",
      "[896]\tvalid_0's cross_entropy: 0.43938\n",
      "[897]\tvalid_0's cross_entropy: 0.439372\n",
      "[898]\tvalid_0's cross_entropy: 0.439365\n",
      "[899]\tvalid_0's cross_entropy: 0.439365\n",
      "[900]\tvalid_0's cross_entropy: 0.439357\n",
      "[901]\tvalid_0's cross_entropy: 0.439352\n",
      "[902]\tvalid_0's cross_entropy: 0.43935\n",
      "[903]\tvalid_0's cross_entropy: 0.439352\n",
      "[904]\tvalid_0's cross_entropy: 0.439347\n",
      "[905]\tvalid_0's cross_entropy: 0.439344\n",
      "[906]\tvalid_0's cross_entropy: 0.439349\n",
      "[907]\tvalid_0's cross_entropy: 0.439349\n",
      "[908]\tvalid_0's cross_entropy: 0.439345\n",
      "[909]\tvalid_0's cross_entropy: 0.439345\n",
      "[910]\tvalid_0's cross_entropy: 0.439342\n",
      "[911]\tvalid_0's cross_entropy: 0.43934\n",
      "[912]\tvalid_0's cross_entropy: 0.439344\n",
      "[913]\tvalid_0's cross_entropy: 0.439338\n",
      "[914]\tvalid_0's cross_entropy: 0.439347\n",
      "[915]\tvalid_0's cross_entropy: 0.439342\n",
      "[916]\tvalid_0's cross_entropy: 0.439337\n",
      "[917]\tvalid_0's cross_entropy: 0.439332\n",
      "[918]\tvalid_0's cross_entropy: 0.439339\n",
      "[919]\tvalid_0's cross_entropy: 0.439343\n",
      "[920]\tvalid_0's cross_entropy: 0.439339\n",
      "[921]\tvalid_0's cross_entropy: 0.439349\n",
      "[922]\tvalid_0's cross_entropy: 0.439354\n",
      "[923]\tvalid_0's cross_entropy: 0.43935\n",
      "[924]\tvalid_0's cross_entropy: 0.439349\n",
      "[925]\tvalid_0's cross_entropy: 0.439349\n",
      "[926]\tvalid_0's cross_entropy: 0.439343\n",
      "[927]\tvalid_0's cross_entropy: 0.439339\n",
      "[928]\tvalid_0's cross_entropy: 0.439336\n",
      "[929]\tvalid_0's cross_entropy: 0.439334\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[930]\tvalid_0's cross_entropy: 0.439326\n",
      "[931]\tvalid_0's cross_entropy: 0.439323\n",
      "[932]\tvalid_0's cross_entropy: 0.439324\n",
      "[933]\tvalid_0's cross_entropy: 0.439331\n",
      "[934]\tvalid_0's cross_entropy: 0.439329\n",
      "[935]\tvalid_0's cross_entropy: 0.439321\n",
      "[936]\tvalid_0's cross_entropy: 0.439317\n",
      "[937]\tvalid_0's cross_entropy: 0.439314\n",
      "[938]\tvalid_0's cross_entropy: 0.439312\n",
      "[939]\tvalid_0's cross_entropy: 0.439316\n",
      "[940]\tvalid_0's cross_entropy: 0.43931\n",
      "[941]\tvalid_0's cross_entropy: 0.439307\n",
      "[942]\tvalid_0's cross_entropy: 0.439305\n",
      "[943]\tvalid_0's cross_entropy: 0.439301\n",
      "[944]\tvalid_0's cross_entropy: 0.439297\n",
      "[945]\tvalid_0's cross_entropy: 0.439292\n",
      "[946]\tvalid_0's cross_entropy: 0.439292\n",
      "[947]\tvalid_0's cross_entropy: 0.439288\n",
      "[948]\tvalid_0's cross_entropy: 0.439289\n",
      "[949]\tvalid_0's cross_entropy: 0.439284\n",
      "[950]\tvalid_0's cross_entropy: 0.439282\n",
      "[951]\tvalid_0's cross_entropy: 0.439276\n",
      "[952]\tvalid_0's cross_entropy: 0.439278\n",
      "[953]\tvalid_0's cross_entropy: 0.439276\n",
      "[954]\tvalid_0's cross_entropy: 0.439267\n",
      "[955]\tvalid_0's cross_entropy: 0.439266\n",
      "[956]\tvalid_0's cross_entropy: 0.43926\n",
      "[957]\tvalid_0's cross_entropy: 0.439255\n",
      "[958]\tvalid_0's cross_entropy: 0.439255\n",
      "[959]\tvalid_0's cross_entropy: 0.43926\n",
      "[960]\tvalid_0's cross_entropy: 0.439259\n",
      "[961]\tvalid_0's cross_entropy: 0.439254\n",
      "[962]\tvalid_0's cross_entropy: 0.439257\n",
      "[963]\tvalid_0's cross_entropy: 0.439254\n",
      "[964]\tvalid_0's cross_entropy: 0.439259\n",
      "[965]\tvalid_0's cross_entropy: 0.439251\n",
      "[966]\tvalid_0's cross_entropy: 0.439249\n",
      "[967]\tvalid_0's cross_entropy: 0.43925\n",
      "[968]\tvalid_0's cross_entropy: 0.439249\n",
      "[969]\tvalid_0's cross_entropy: 0.439256\n",
      "[970]\tvalid_0's cross_entropy: 0.439257\n",
      "[971]\tvalid_0's cross_entropy: 0.439259\n",
      "[972]\tvalid_0's cross_entropy: 0.439257\n",
      "[973]\tvalid_0's cross_entropy: 0.43926\n",
      "[974]\tvalid_0's cross_entropy: 0.439256\n",
      "[975]\tvalid_0's cross_entropy: 0.439258\n",
      "[976]\tvalid_0's cross_entropy: 0.439263\n",
      "[977]\tvalid_0's cross_entropy: 0.439267\n",
      "[978]\tvalid_0's cross_entropy: 0.439267\n",
      "[979]\tvalid_0's cross_entropy: 0.439273\n",
      "[980]\tvalid_0's cross_entropy: 0.439264\n",
      "[981]\tvalid_0's cross_entropy: 0.439268\n",
      "[982]\tvalid_0's cross_entropy: 0.439268\n",
      "[983]\tvalid_0's cross_entropy: 0.43926\n",
      "[984]\tvalid_0's cross_entropy: 0.439257\n",
      "[985]\tvalid_0's cross_entropy: 0.439257\n",
      "[986]\tvalid_0's cross_entropy: 0.439252\n",
      "[987]\tvalid_0's cross_entropy: 0.439252\n",
      "[988]\tvalid_0's cross_entropy: 0.439246\n",
      "[989]\tvalid_0's cross_entropy: 0.43924\n",
      "[990]\tvalid_0's cross_entropy: 0.439242\n",
      "[991]\tvalid_0's cross_entropy: 0.439232\n",
      "[992]\tvalid_0's cross_entropy: 0.43923\n",
      "[993]\tvalid_0's cross_entropy: 0.439228\n",
      "[994]\tvalid_0's cross_entropy: 0.439227\n",
      "[995]\tvalid_0's cross_entropy: 0.439228\n",
      "[996]\tvalid_0's cross_entropy: 0.439228\n",
      "[997]\tvalid_0's cross_entropy: 0.43923\n",
      "[998]\tvalid_0's cross_entropy: 0.439234\n",
      "[999]\tvalid_0's cross_entropy: 0.439232\n",
      "[1000]\tvalid_0's cross_entropy: 0.439231\n",
      "[1001]\tvalid_0's cross_entropy: 0.43923\n",
      "[1002]\tvalid_0's cross_entropy: 0.439224\n",
      "[1003]\tvalid_0's cross_entropy: 0.439224\n",
      "[1004]\tvalid_0's cross_entropy: 0.439227\n",
      "[1005]\tvalid_0's cross_entropy: 0.439225\n",
      "[1006]\tvalid_0's cross_entropy: 0.439221\n",
      "[1007]\tvalid_0's cross_entropy: 0.439221\n",
      "[1008]\tvalid_0's cross_entropy: 0.439217\n",
      "[1009]\tvalid_0's cross_entropy: 0.439214\n",
      "[1010]\tvalid_0's cross_entropy: 0.439209\n",
      "[1011]\tvalid_0's cross_entropy: 0.43921\n",
      "[1012]\tvalid_0's cross_entropy: 0.439214\n",
      "[1013]\tvalid_0's cross_entropy: 0.439209\n",
      "[1014]\tvalid_0's cross_entropy: 0.439214\n",
      "[1015]\tvalid_0's cross_entropy: 0.43921\n",
      "[1016]\tvalid_0's cross_entropy: 0.439211\n",
      "[1017]\tvalid_0's cross_entropy: 0.43921\n",
      "[1018]\tvalid_0's cross_entropy: 0.439218\n",
      "[1019]\tvalid_0's cross_entropy: 0.439213\n",
      "[1020]\tvalid_0's cross_entropy: 0.439209\n",
      "[1021]\tvalid_0's cross_entropy: 0.439205\n",
      "[1022]\tvalid_0's cross_entropy: 0.439209\n",
      "[1023]\tvalid_0's cross_entropy: 0.439207\n",
      "[1024]\tvalid_0's cross_entropy: 0.439211\n",
      "[1025]\tvalid_0's cross_entropy: 0.439214\n",
      "[1026]\tvalid_0's cross_entropy: 0.439207\n",
      "[1027]\tvalid_0's cross_entropy: 0.439201\n",
      "[1028]\tvalid_0's cross_entropy: 0.439192\n",
      "[1029]\tvalid_0's cross_entropy: 0.439189\n",
      "[1030]\tvalid_0's cross_entropy: 0.439191\n",
      "[1031]\tvalid_0's cross_entropy: 0.439193\n",
      "[1032]\tvalid_0's cross_entropy: 0.439195\n",
      "[1033]\tvalid_0's cross_entropy: 0.439195\n",
      "[1034]\tvalid_0's cross_entropy: 0.439186\n",
      "[1035]\tvalid_0's cross_entropy: 0.43919\n",
      "[1036]\tvalid_0's cross_entropy: 0.439188\n",
      "[1037]\tvalid_0's cross_entropy: 0.439181\n",
      "[1038]\tvalid_0's cross_entropy: 0.43918\n",
      "[1039]\tvalid_0's cross_entropy: 0.439182\n",
      "[1040]\tvalid_0's cross_entropy: 0.43918\n",
      "[1041]\tvalid_0's cross_entropy: 0.439181\n",
      "[1042]\tvalid_0's cross_entropy: 0.43919\n",
      "[1043]\tvalid_0's cross_entropy: 0.439188\n",
      "[1044]\tvalid_0's cross_entropy: 0.439194\n",
      "[1045]\tvalid_0's cross_entropy: 0.439196\n",
      "[1046]\tvalid_0's cross_entropy: 0.439195\n",
      "[1047]\tvalid_0's cross_entropy: 0.439187\n",
      "[1048]\tvalid_0's cross_entropy: 0.43919\n",
      "[1049]\tvalid_0's cross_entropy: 0.439186\n",
      "[1050]\tvalid_0's cross_entropy: 0.439185\n",
      "[1051]\tvalid_0's cross_entropy: 0.439188\n",
      "[1052]\tvalid_0's cross_entropy: 0.43919\n",
      "[1053]\tvalid_0's cross_entropy: 0.439191\n",
      "[1054]\tvalid_0's cross_entropy: 0.439189\n",
      "[1055]\tvalid_0's cross_entropy: 0.439186\n",
      "[1056]\tvalid_0's cross_entropy: 0.439186\n",
      "[1057]\tvalid_0's cross_entropy: 0.439177\n",
      "[1058]\tvalid_0's cross_entropy: 0.439175\n",
      "[1059]\tvalid_0's cross_entropy: 0.439175\n",
      "[1060]\tvalid_0's cross_entropy: 0.439169\n",
      "[1061]\tvalid_0's cross_entropy: 0.439166\n",
      "[1062]\tvalid_0's cross_entropy: 0.439164\n",
      "[1063]\tvalid_0's cross_entropy: 0.439158\n",
      "[1064]\tvalid_0's cross_entropy: 0.439154\n",
      "[1065]\tvalid_0's cross_entropy: 0.439153\n",
      "[1066]\tvalid_0's cross_entropy: 0.439148\n",
      "[1067]\tvalid_0's cross_entropy: 0.439147\n",
      "[1068]\tvalid_0's cross_entropy: 0.439148\n",
      "[1069]\tvalid_0's cross_entropy: 0.439149\n",
      "[1070]\tvalid_0's cross_entropy: 0.43915\n",
      "[1071]\tvalid_0's cross_entropy: 0.439149\n",
      "[1072]\tvalid_0's cross_entropy: 0.43914\n",
      "[1073]\tvalid_0's cross_entropy: 0.439137\n",
      "[1074]\tvalid_0's cross_entropy: 0.439135\n",
      "[1075]\tvalid_0's cross_entropy: 0.439135\n",
      "[1076]\tvalid_0's cross_entropy: 0.439135\n",
      "[1077]\tvalid_0's cross_entropy: 0.439135\n",
      "[1078]\tvalid_0's cross_entropy: 0.439133\n",
      "[1079]\tvalid_0's cross_entropy: 0.439136\n",
      "[1080]\tvalid_0's cross_entropy: 0.439139\n",
      "[1081]\tvalid_0's cross_entropy: 0.439137\n",
      "[1082]\tvalid_0's cross_entropy: 0.439129\n",
      "[1083]\tvalid_0's cross_entropy: 0.439134\n",
      "[1084]\tvalid_0's cross_entropy: 0.439139\n",
      "[1085]\tvalid_0's cross_entropy: 0.439147\n",
      "[1086]\tvalid_0's cross_entropy: 0.439137\n",
      "[1087]\tvalid_0's cross_entropy: 0.439141\n",
      "[1088]\tvalid_0's cross_entropy: 0.439146\n",
      "[1089]\tvalid_0's cross_entropy: 0.439148\n",
      "[1090]\tvalid_0's cross_entropy: 0.439157\n",
      "[1091]\tvalid_0's cross_entropy: 0.439147\n",
      "[1092]\tvalid_0's cross_entropy: 0.439148\n",
      "[1093]\tvalid_0's cross_entropy: 0.439143\n",
      "[1094]\tvalid_0's cross_entropy: 0.439146\n",
      "[1095]\tvalid_0's cross_entropy: 0.439148\n",
      "[1096]\tvalid_0's cross_entropy: 0.439147\n",
      "[1097]\tvalid_0's cross_entropy: 0.43914\n",
      "[1098]\tvalid_0's cross_entropy: 0.439138\n",
      "[1099]\tvalid_0's cross_entropy: 0.439132\n",
      "[1100]\tvalid_0's cross_entropy: 0.439131\n",
      "[1101]\tvalid_0's cross_entropy: 0.439132\n",
      "[1102]\tvalid_0's cross_entropy: 0.43913\n",
      "[1103]\tvalid_0's cross_entropy: 0.439127\n",
      "[1104]\tvalid_0's cross_entropy: 0.439128\n",
      "[1105]\tvalid_0's cross_entropy: 0.439133\n",
      "[1106]\tvalid_0's cross_entropy: 0.439132\n",
      "[1107]\tvalid_0's cross_entropy: 0.439141\n",
      "[1108]\tvalid_0's cross_entropy: 0.43914\n",
      "[1109]\tvalid_0's cross_entropy: 0.439139\n",
      "[1110]\tvalid_0's cross_entropy: 0.439135\n",
      "[1111]\tvalid_0's cross_entropy: 0.439137\n",
      "[1112]\tvalid_0's cross_entropy: 0.439135\n",
      "[1113]\tvalid_0's cross_entropy: 0.43914\n",
      "[1114]\tvalid_0's cross_entropy: 0.439136\n",
      "[1115]\tvalid_0's cross_entropy: 0.439138\n",
      "[1116]\tvalid_0's cross_entropy: 0.439138\n",
      "[1117]\tvalid_0's cross_entropy: 0.43914\n",
      "[1118]\tvalid_0's cross_entropy: 0.439146\n",
      "[1119]\tvalid_0's cross_entropy: 0.439135\n",
      "[1120]\tvalid_0's cross_entropy: 0.439131\n",
      "[1121]\tvalid_0's cross_entropy: 0.439136\n",
      "[1122]\tvalid_0's cross_entropy: 0.439139\n",
      "[1123]\tvalid_0's cross_entropy: 0.43914\n",
      "[1124]\tvalid_0's cross_entropy: 0.439138\n",
      "[1125]\tvalid_0's cross_entropy: 0.439145\n",
      "[1126]\tvalid_0's cross_entropy: 0.439153\n",
      "[1127]\tvalid_0's cross_entropy: 0.439153\n",
      "[1128]\tvalid_0's cross_entropy: 0.439152\n",
      "[1129]\tvalid_0's cross_entropy: 0.439153\n",
      "[1130]\tvalid_0's cross_entropy: 0.439152\n",
      "[1131]\tvalid_0's cross_entropy: 0.439154\n",
      "[1132]\tvalid_0's cross_entropy: 0.439153\n",
      "[1133]\tvalid_0's cross_entropy: 0.43916\n",
      "[1134]\tvalid_0's cross_entropy: 0.439157\n",
      "[1135]\tvalid_0's cross_entropy: 0.439159\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1136]\tvalid_0's cross_entropy: 0.439154\n",
      "[1137]\tvalid_0's cross_entropy: 0.439157\n",
      "[1138]\tvalid_0's cross_entropy: 0.439154\n",
      "[1139]\tvalid_0's cross_entropy: 0.439155\n",
      "[1140]\tvalid_0's cross_entropy: 0.439161\n",
      "[1141]\tvalid_0's cross_entropy: 0.439152\n",
      "[1142]\tvalid_0's cross_entropy: 0.439154\n",
      "[1143]\tvalid_0's cross_entropy: 0.439154\n",
      "[1144]\tvalid_0's cross_entropy: 0.439148\n",
      "[1145]\tvalid_0's cross_entropy: 0.439152\n",
      "[1146]\tvalid_0's cross_entropy: 0.439149\n",
      "[1147]\tvalid_0's cross_entropy: 0.439146\n",
      "[1148]\tvalid_0's cross_entropy: 0.439144\n",
      "[1149]\tvalid_0's cross_entropy: 0.439148\n",
      "[1150]\tvalid_0's cross_entropy: 0.439149\n",
      "[1151]\tvalid_0's cross_entropy: 0.43915\n",
      "[1152]\tvalid_0's cross_entropy: 0.439154\n",
      "[1153]\tvalid_0's cross_entropy: 0.439147\n",
      "[1154]\tvalid_0's cross_entropy: 0.439147\n",
      "[1155]\tvalid_0's cross_entropy: 0.439147\n",
      "[1156]\tvalid_0's cross_entropy: 0.439152\n",
      "[1157]\tvalid_0's cross_entropy: 0.439154\n",
      "[1158]\tvalid_0's cross_entropy: 0.439146\n",
      "[1159]\tvalid_0's cross_entropy: 0.43915\n",
      "[1160]\tvalid_0's cross_entropy: 0.439156\n",
      "[1161]\tvalid_0's cross_entropy: 0.43916\n",
      "[1162]\tvalid_0's cross_entropy: 0.439154\n",
      "[1163]\tvalid_0's cross_entropy: 0.439158\n",
      "[1164]\tvalid_0's cross_entropy: 0.439158\n",
      "[1165]\tvalid_0's cross_entropy: 0.439159\n",
      "[1166]\tvalid_0's cross_entropy: 0.439159\n",
      "[1167]\tvalid_0's cross_entropy: 0.439155\n",
      "[1168]\tvalid_0's cross_entropy: 0.439152\n",
      "[1169]\tvalid_0's cross_entropy: 0.439152\n",
      "[1170]\tvalid_0's cross_entropy: 0.439158\n",
      "[1171]\tvalid_0's cross_entropy: 0.439157\n",
      "[1172]\tvalid_0's cross_entropy: 0.43916\n",
      "[1173]\tvalid_0's cross_entropy: 0.439165\n",
      "[1174]\tvalid_0's cross_entropy: 0.439169\n",
      "[1175]\tvalid_0's cross_entropy: 0.439173\n",
      "[1176]\tvalid_0's cross_entropy: 0.439175\n",
      "[1177]\tvalid_0's cross_entropy: 0.439172\n",
      "[1178]\tvalid_0's cross_entropy: 0.439175\n",
      "[1179]\tvalid_0's cross_entropy: 0.439175\n",
      "[1180]\tvalid_0's cross_entropy: 0.439177\n",
      "[1181]\tvalid_0's cross_entropy: 0.439176\n",
      "[1182]\tvalid_0's cross_entropy: 0.439178\n",
      "[1183]\tvalid_0's cross_entropy: 0.439177\n",
      "[1184]\tvalid_0's cross_entropy: 0.439174\n",
      "[1185]\tvalid_0's cross_entropy: 0.439175\n",
      "[1186]\tvalid_0's cross_entropy: 0.439176\n",
      "[1187]\tvalid_0's cross_entropy: 0.439184\n",
      "[1188]\tvalid_0's cross_entropy: 0.439189\n",
      "[1189]\tvalid_0's cross_entropy: 0.439189\n",
      "[1190]\tvalid_0's cross_entropy: 0.439187\n",
      "[1191]\tvalid_0's cross_entropy: 0.439186\n",
      "[1192]\tvalid_0's cross_entropy: 0.439186\n",
      "[1193]\tvalid_0's cross_entropy: 0.439187\n",
      "[1194]\tvalid_0's cross_entropy: 0.439185\n",
      "[1195]\tvalid_0's cross_entropy: 0.439186\n",
      "[1196]\tvalid_0's cross_entropy: 0.439189\n",
      "[1197]\tvalid_0's cross_entropy: 0.43919\n",
      "[1198]\tvalid_0's cross_entropy: 0.439195\n",
      "[1199]\tvalid_0's cross_entropy: 0.439194\n",
      "[1200]\tvalid_0's cross_entropy: 0.43919\n",
      "[1201]\tvalid_0's cross_entropy: 0.439187\n",
      "[1202]\tvalid_0's cross_entropy: 0.439186\n",
      "[1203]\tvalid_0's cross_entropy: 0.439185\n",
      "[1204]\tvalid_0's cross_entropy: 0.439185\n",
      "[1205]\tvalid_0's cross_entropy: 0.439181\n",
      "[1206]\tvalid_0's cross_entropy: 0.439174\n",
      "[1207]\tvalid_0's cross_entropy: 0.439178\n",
      "[1208]\tvalid_0's cross_entropy: 0.439179\n",
      "[1209]\tvalid_0's cross_entropy: 0.439179\n",
      "[1210]\tvalid_0's cross_entropy: 0.439178\n",
      "[1211]\tvalid_0's cross_entropy: 0.439176\n",
      "[1212]\tvalid_0's cross_entropy: 0.439178\n",
      "[1213]\tvalid_0's cross_entropy: 0.439174\n",
      "[1214]\tvalid_0's cross_entropy: 0.43918\n",
      "[1215]\tvalid_0's cross_entropy: 0.439187\n",
      "[1216]\tvalid_0's cross_entropy: 0.439184\n",
      "[1217]\tvalid_0's cross_entropy: 0.439187\n",
      "[1218]\tvalid_0's cross_entropy: 0.439185\n",
      "[1219]\tvalid_0's cross_entropy: 0.43918\n",
      "[1220]\tvalid_0's cross_entropy: 0.439175\n",
      "[1221]\tvalid_0's cross_entropy: 0.439175\n",
      "[1222]\tvalid_0's cross_entropy: 0.43918\n",
      "[1223]\tvalid_0's cross_entropy: 0.439175\n",
      "[1224]\tvalid_0's cross_entropy: 0.439179\n",
      "[1225]\tvalid_0's cross_entropy: 0.439182\n",
      "[1226]\tvalid_0's cross_entropy: 0.439191\n",
      "[1227]\tvalid_0's cross_entropy: 0.439192\n",
      "[1228]\tvalid_0's cross_entropy: 0.439191\n",
      "[1229]\tvalid_0's cross_entropy: 0.439185\n",
      "[1230]\tvalid_0's cross_entropy: 0.439189\n",
      "[1231]\tvalid_0's cross_entropy: 0.439193\n",
      "[1232]\tvalid_0's cross_entropy: 0.439202\n",
      "[1233]\tvalid_0's cross_entropy: 0.439206\n",
      "[1234]\tvalid_0's cross_entropy: 0.439205\n",
      "[1235]\tvalid_0's cross_entropy: 0.439209\n",
      "[1236]\tvalid_0's cross_entropy: 0.439204\n",
      "[1237]\tvalid_0's cross_entropy: 0.439205\n",
      "[1238]\tvalid_0's cross_entropy: 0.439212\n",
      "[1239]\tvalid_0's cross_entropy: 0.43921\n",
      "[1240]\tvalid_0's cross_entropy: 0.439207\n",
      "[1241]\tvalid_0's cross_entropy: 0.439205\n",
      "[1242]\tvalid_0's cross_entropy: 0.43921\n",
      "[1243]\tvalid_0's cross_entropy: 0.439211\n",
      "[1244]\tvalid_0's cross_entropy: 0.439208\n",
      "[1245]\tvalid_0's cross_entropy: 0.439214\n",
      "[1246]\tvalid_0's cross_entropy: 0.439211\n",
      "[1247]\tvalid_0's cross_entropy: 0.439211\n",
      "[1248]\tvalid_0's cross_entropy: 0.439213\n",
      "[1249]\tvalid_0's cross_entropy: 0.439211\n",
      "[1250]\tvalid_0's cross_entropy: 0.43921\n",
      "[1251]\tvalid_0's cross_entropy: 0.439211\n",
      "[1252]\tvalid_0's cross_entropy: 0.439218\n",
      "[1253]\tvalid_0's cross_entropy: 0.439222\n",
      "[1254]\tvalid_0's cross_entropy: 0.439226\n",
      "[1255]\tvalid_0's cross_entropy: 0.439222\n",
      "[1256]\tvalid_0's cross_entropy: 0.439219\n",
      "[1257]\tvalid_0's cross_entropy: 0.439219\n",
      "[1258]\tvalid_0's cross_entropy: 0.439223\n",
      "[1259]\tvalid_0's cross_entropy: 0.439225\n",
      "[1260]\tvalid_0's cross_entropy: 0.439229\n",
      "[1261]\tvalid_0's cross_entropy: 0.439229\n",
      "[1262]\tvalid_0's cross_entropy: 0.439223\n",
      "[1263]\tvalid_0's cross_entropy: 0.439207\n",
      "[1264]\tvalid_0's cross_entropy: 0.439204\n",
      "[1265]\tvalid_0's cross_entropy: 0.439201\n",
      "[1266]\tvalid_0's cross_entropy: 0.439211\n",
      "[1267]\tvalid_0's cross_entropy: 0.439209\n",
      "[1268]\tvalid_0's cross_entropy: 0.439216\n",
      "[1269]\tvalid_0's cross_entropy: 0.439216\n",
      "[1270]\tvalid_0's cross_entropy: 0.43922\n",
      "[1271]\tvalid_0's cross_entropy: 0.439223\n",
      "[1272]\tvalid_0's cross_entropy: 0.43923\n",
      "[1273]\tvalid_0's cross_entropy: 0.439227\n",
      "[1274]\tvalid_0's cross_entropy: 0.439228\n",
      "[1275]\tvalid_0's cross_entropy: 0.439233\n",
      "[1276]\tvalid_0's cross_entropy: 0.439232\n",
      "[1277]\tvalid_0's cross_entropy: 0.439242\n",
      "[1278]\tvalid_0's cross_entropy: 0.439237\n",
      "[1279]\tvalid_0's cross_entropy: 0.439239\n",
      "[1280]\tvalid_0's cross_entropy: 0.439233\n",
      "[1281]\tvalid_0's cross_entropy: 0.439235\n",
      "[1282]\tvalid_0's cross_entropy: 0.439239\n",
      "[1283]\tvalid_0's cross_entropy: 0.439246\n",
      "[1284]\tvalid_0's cross_entropy: 0.439244\n",
      "[1285]\tvalid_0's cross_entropy: 0.439241\n",
      "[1286]\tvalid_0's cross_entropy: 0.439243\n",
      "[1287]\tvalid_0's cross_entropy: 0.439244\n",
      "[1288]\tvalid_0's cross_entropy: 0.43924\n",
      "[1289]\tvalid_0's cross_entropy: 0.43924\n",
      "[1290]\tvalid_0's cross_entropy: 0.439236\n",
      "[1291]\tvalid_0's cross_entropy: 0.439241\n",
      "[1292]\tvalid_0's cross_entropy: 0.439247\n",
      "[1293]\tvalid_0's cross_entropy: 0.439246\n",
      "[1294]\tvalid_0's cross_entropy: 0.439248\n",
      "[1295]\tvalid_0's cross_entropy: 0.439247\n",
      "[1296]\tvalid_0's cross_entropy: 0.43925\n",
      "[1297]\tvalid_0's cross_entropy: 0.439249\n",
      "[1298]\tvalid_0's cross_entropy: 0.439252\n",
      "[1299]\tvalid_0's cross_entropy: 0.439257\n",
      "[1300]\tvalid_0's cross_entropy: 0.439249\n",
      "[1301]\tvalid_0's cross_entropy: 0.43925\n",
      "[1302]\tvalid_0's cross_entropy: 0.439254\n",
      "[1303]\tvalid_0's cross_entropy: 0.439259\n",
      "Early stopping, best iteration is:\n",
      "[1103]\tvalid_0's cross_entropy: 0.439127\n",
      "[LightGBM] [Info] Number of positive: 21236, number of negative: 98764\n",
      "[LightGBM] [Info] [cross_entropy:Init]: (metric) labels passed interval [0, 1] check\n",
      "[LightGBM] [Info] [cross_entropy:Init]: sum-of-weights = 120000.000000\n",
      "[LightGBM] [Warning] Auto-choosing row-wise multi-threading, the overhead of testing was 0.019606 seconds.\n",
      "You can set `force_row_wise=true` to remove the overhead.\n",
      "And if memory is not enough, you can set `force_col_wise=true`.\n",
      "[LightGBM] [Info] Total Bins 6602\n",
      "[LightGBM] [Info] Number of data points in the train set: 120000, number of used features: 47\n",
      "[LightGBM] [Info] [cross_entropy:Init]: (metric) labels passed interval [0, 1] check\n",
      "[LightGBM] [Info] [cross_entropy:Init]: sum-of-weights = 30000.000000\n",
      "[LightGBM] [Info] [binary:BoostFromScore]: pavg=0.176967 -> initscore=-1.537035\n",
      "[LightGBM] [Info] Start training from score -1.537035\n",
      "[1]\tvalid_0's cross_entropy: 0.466435\n",
      "Training until validation scores don't improve for 200 rounds\n",
      "[2]\tvalid_0's cross_entropy: 0.466125\n",
      "[3]\tvalid_0's cross_entropy: 0.465855\n",
      "[4]\tvalid_0's cross_entropy: 0.465561\n",
      "[5]\tvalid_0's cross_entropy: 0.465267\n",
      "[6]\tvalid_0's cross_entropy: 0.465009\n",
      "[7]\tvalid_0's cross_entropy: 0.464761\n",
      "[8]\tvalid_0's cross_entropy: 0.464477\n",
      "[9]\tvalid_0's cross_entropy: 0.464207\n",
      "[10]\tvalid_0's cross_entropy: 0.463927\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[11]\tvalid_0's cross_entropy: 0.463665\n",
      "[12]\tvalid_0's cross_entropy: 0.463421\n",
      "[13]\tvalid_0's cross_entropy: 0.463177\n",
      "[14]\tvalid_0's cross_entropy: 0.462943\n",
      "[15]\tvalid_0's cross_entropy: 0.462708\n",
      "[16]\tvalid_0's cross_entropy: 0.462472\n",
      "[17]\tvalid_0's cross_entropy: 0.462247\n",
      "[18]\tvalid_0's cross_entropy: 0.462007\n",
      "[19]\tvalid_0's cross_entropy: 0.461803\n",
      "[20]\tvalid_0's cross_entropy: 0.461581\n",
      "[21]\tvalid_0's cross_entropy: 0.461357\n",
      "[22]\tvalid_0's cross_entropy: 0.461156\n",
      "[23]\tvalid_0's cross_entropy: 0.46096\n",
      "[24]\tvalid_0's cross_entropy: 0.460764\n",
      "[25]\tvalid_0's cross_entropy: 0.460558\n",
      "[26]\tvalid_0's cross_entropy: 0.460353\n",
      "[27]\tvalid_0's cross_entropy: 0.460162\n",
      "[28]\tvalid_0's cross_entropy: 0.459992\n",
      "[29]\tvalid_0's cross_entropy: 0.459805\n",
      "[30]\tvalid_0's cross_entropy: 0.459626\n",
      "[31]\tvalid_0's cross_entropy: 0.45944\n",
      "[32]\tvalid_0's cross_entropy: 0.459265\n",
      "[33]\tvalid_0's cross_entropy: 0.459079\n",
      "[34]\tvalid_0's cross_entropy: 0.458897\n",
      "[35]\tvalid_0's cross_entropy: 0.458721\n",
      "[36]\tvalid_0's cross_entropy: 0.458579\n",
      "[37]\tvalid_0's cross_entropy: 0.458404\n",
      "[38]\tvalid_0's cross_entropy: 0.458234\n",
      "[39]\tvalid_0's cross_entropy: 0.458063\n",
      "[40]\tvalid_0's cross_entropy: 0.457894\n",
      "[41]\tvalid_0's cross_entropy: 0.457741\n",
      "[42]\tvalid_0's cross_entropy: 0.457591\n",
      "[43]\tvalid_0's cross_entropy: 0.457437\n",
      "[44]\tvalid_0's cross_entropy: 0.457287\n",
      "[45]\tvalid_0's cross_entropy: 0.45714\n",
      "[46]\tvalid_0's cross_entropy: 0.456983\n",
      "[47]\tvalid_0's cross_entropy: 0.456852\n",
      "[48]\tvalid_0's cross_entropy: 0.456687\n",
      "[49]\tvalid_0's cross_entropy: 0.45656\n",
      "[50]\tvalid_0's cross_entropy: 0.456412\n",
      "[51]\tvalid_0's cross_entropy: 0.456275\n",
      "[52]\tvalid_0's cross_entropy: 0.456128\n",
      "[53]\tvalid_0's cross_entropy: 0.455998\n",
      "[54]\tvalid_0's cross_entropy: 0.455862\n",
      "[55]\tvalid_0's cross_entropy: 0.455733\n",
      "[56]\tvalid_0's cross_entropy: 0.455608\n",
      "[57]\tvalid_0's cross_entropy: 0.455487\n",
      "[58]\tvalid_0's cross_entropy: 0.455358\n",
      "[59]\tvalid_0's cross_entropy: 0.455237\n",
      "[60]\tvalid_0's cross_entropy: 0.455125\n",
      "[61]\tvalid_0's cross_entropy: 0.454989\n",
      "[62]\tvalid_0's cross_entropy: 0.454866\n",
      "[63]\tvalid_0's cross_entropy: 0.454746\n",
      "[64]\tvalid_0's cross_entropy: 0.454645\n",
      "[65]\tvalid_0's cross_entropy: 0.454524\n",
      "[66]\tvalid_0's cross_entropy: 0.454419\n",
      "[67]\tvalid_0's cross_entropy: 0.454314\n",
      "[68]\tvalid_0's cross_entropy: 0.454197\n",
      "[69]\tvalid_0's cross_entropy: 0.454087\n",
      "[70]\tvalid_0's cross_entropy: 0.45398\n",
      "[71]\tvalid_0's cross_entropy: 0.453856\n",
      "[72]\tvalid_0's cross_entropy: 0.453756\n",
      "[73]\tvalid_0's cross_entropy: 0.453668\n",
      "[74]\tvalid_0's cross_entropy: 0.453564\n",
      "[75]\tvalid_0's cross_entropy: 0.453461\n",
      "[76]\tvalid_0's cross_entropy: 0.453364\n",
      "[77]\tvalid_0's cross_entropy: 0.45326\n",
      "[78]\tvalid_0's cross_entropy: 0.453169\n",
      "[79]\tvalid_0's cross_entropy: 0.453074\n",
      "[80]\tvalid_0's cross_entropy: 0.452974\n",
      "[81]\tvalid_0's cross_entropy: 0.452892\n",
      "[82]\tvalid_0's cross_entropy: 0.452805\n",
      "[83]\tvalid_0's cross_entropy: 0.452708\n",
      "[84]\tvalid_0's cross_entropy: 0.452615\n",
      "[85]\tvalid_0's cross_entropy: 0.452526\n",
      "[86]\tvalid_0's cross_entropy: 0.452436\n",
      "[87]\tvalid_0's cross_entropy: 0.452339\n",
      "[88]\tvalid_0's cross_entropy: 0.452233\n",
      "[89]\tvalid_0's cross_entropy: 0.452142\n",
      "[90]\tvalid_0's cross_entropy: 0.45205\n",
      "[91]\tvalid_0's cross_entropy: 0.451957\n",
      "[92]\tvalid_0's cross_entropy: 0.451877\n",
      "[93]\tvalid_0's cross_entropy: 0.451794\n",
      "[94]\tvalid_0's cross_entropy: 0.45172\n",
      "[95]\tvalid_0's cross_entropy: 0.451623\n",
      "[96]\tvalid_0's cross_entropy: 0.451544\n",
      "[97]\tvalid_0's cross_entropy: 0.45146\n",
      "[98]\tvalid_0's cross_entropy: 0.451375\n",
      "[99]\tvalid_0's cross_entropy: 0.451301\n",
      "[100]\tvalid_0's cross_entropy: 0.451224\n",
      "[101]\tvalid_0's cross_entropy: 0.451156\n",
      "[102]\tvalid_0's cross_entropy: 0.451091\n",
      "[103]\tvalid_0's cross_entropy: 0.451025\n",
      "[104]\tvalid_0's cross_entropy: 0.450961\n",
      "[105]\tvalid_0's cross_entropy: 0.450905\n",
      "[106]\tvalid_0's cross_entropy: 0.450839\n",
      "[107]\tvalid_0's cross_entropy: 0.450767\n",
      "[108]\tvalid_0's cross_entropy: 0.450711\n",
      "[109]\tvalid_0's cross_entropy: 0.450648\n",
      "[110]\tvalid_0's cross_entropy: 0.450581\n",
      "[111]\tvalid_0's cross_entropy: 0.450525\n",
      "[112]\tvalid_0's cross_entropy: 0.450459\n",
      "[113]\tvalid_0's cross_entropy: 0.450399\n",
      "[114]\tvalid_0's cross_entropy: 0.450332\n",
      "[115]\tvalid_0's cross_entropy: 0.450271\n",
      "[116]\tvalid_0's cross_entropy: 0.450209\n",
      "[117]\tvalid_0's cross_entropy: 0.450142\n",
      "[118]\tvalid_0's cross_entropy: 0.450082\n",
      "[119]\tvalid_0's cross_entropy: 0.450014\n",
      "[120]\tvalid_0's cross_entropy: 0.449945\n",
      "[121]\tvalid_0's cross_entropy: 0.449894\n",
      "[122]\tvalid_0's cross_entropy: 0.449843\n",
      "[123]\tvalid_0's cross_entropy: 0.449795\n",
      "[124]\tvalid_0's cross_entropy: 0.449725\n",
      "[125]\tvalid_0's cross_entropy: 0.449671\n",
      "[126]\tvalid_0's cross_entropy: 0.44961\n",
      "[127]\tvalid_0's cross_entropy: 0.449544\n",
      "[128]\tvalid_0's cross_entropy: 0.449485\n",
      "[129]\tvalid_0's cross_entropy: 0.449436\n",
      "[130]\tvalid_0's cross_entropy: 0.449379\n",
      "[131]\tvalid_0's cross_entropy: 0.449333\n",
      "[132]\tvalid_0's cross_entropy: 0.449282\n",
      "[133]\tvalid_0's cross_entropy: 0.449212\n",
      "[134]\tvalid_0's cross_entropy: 0.449151\n",
      "[135]\tvalid_0's cross_entropy: 0.449091\n",
      "[136]\tvalid_0's cross_entropy: 0.449051\n",
      "[137]\tvalid_0's cross_entropy: 0.448997\n",
      "[138]\tvalid_0's cross_entropy: 0.448943\n",
      "[139]\tvalid_0's cross_entropy: 0.4489\n",
      "[140]\tvalid_0's cross_entropy: 0.448849\n",
      "[141]\tvalid_0's cross_entropy: 0.448788\n",
      "[142]\tvalid_0's cross_entropy: 0.44875\n",
      "[143]\tvalid_0's cross_entropy: 0.448696\n",
      "[144]\tvalid_0's cross_entropy: 0.448646\n",
      "[145]\tvalid_0's cross_entropy: 0.448601\n",
      "[146]\tvalid_0's cross_entropy: 0.448544\n",
      "[147]\tvalid_0's cross_entropy: 0.448505\n",
      "[148]\tvalid_0's cross_entropy: 0.448462\n",
      "[149]\tvalid_0's cross_entropy: 0.448412\n",
      "[150]\tvalid_0's cross_entropy: 0.448364\n",
      "[151]\tvalid_0's cross_entropy: 0.448313\n",
      "[152]\tvalid_0's cross_entropy: 0.448278\n",
      "[153]\tvalid_0's cross_entropy: 0.448232\n",
      "[154]\tvalid_0's cross_entropy: 0.448188\n",
      "[155]\tvalid_0's cross_entropy: 0.448139\n",
      "[156]\tvalid_0's cross_entropy: 0.448091\n",
      "[157]\tvalid_0's cross_entropy: 0.448045\n",
      "[158]\tvalid_0's cross_entropy: 0.44801\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-17-524191dd6469>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     38\u001b[0m                 early_stopping_rounds=200)\n\u001b[0;32m     39\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 40\u001b[1;33m \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtrain\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtarget\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-1-40c7943bffaf>\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, X, y)\u001b[0m\n\u001b[0;32m     64\u001b[0m                             \u001b[0mnum_boost_round\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnum_boost_round\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     65\u001b[0m                             \u001b[0mvalid_sets\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mlgb_eval\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 66\u001b[1;33m                             early_stopping_rounds=self.early_stopping_rounds)\n\u001b[0m\u001b[0;32m     67\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     68\u001b[0m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstacking_model\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mgbm\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\tools\\conda\\envs\\ml\\lib\\site-packages\\lightgbm\\engine.py\u001b[0m in \u001b[0;36mtrain\u001b[1;34m(params, train_set, num_boost_round, valid_sets, valid_names, fobj, feval, init_model, feature_name, categorical_feature, early_stopping_rounds, evals_result, verbose_eval, learning_rates, keep_training_booster, callbacks)\u001b[0m\n\u001b[0;32m    247\u001b[0m                                     evaluation_result_list=None))\n\u001b[0;32m    248\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 249\u001b[1;33m         \u001b[0mbooster\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfobj\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mfobj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    250\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    251\u001b[0m         \u001b[0mevaluation_result_list\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\tools\\conda\\envs\\ml\\lib\\site-packages\\lightgbm\\basic.py\u001b[0m in \u001b[0;36mupdate\u001b[1;34m(self, train_set, fobj)\u001b[0m\n\u001b[0;32m   2643\u001b[0m             _safe_call(_LIB.LGBM_BoosterUpdateOneIter(\n\u001b[0;32m   2644\u001b[0m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhandle\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2645\u001b[1;33m                 ctypes.byref(is_finished)))\n\u001b[0m\u001b[0;32m   2646\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__is_predicted_cur_iter\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;32mFalse\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0m_\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__num_dataset\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2647\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mis_finished\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalue\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# params = {\n",
    "#         'task': 'train',\n",
    "#         'boosting_type': 'gbdt',\n",
    "#         'objective': 'binary',\n",
    "#         'metric': 'auc',\n",
    "#         'num_leaves': 9,\n",
    "#         'learning_rate': 0.03,\n",
    "#         'feature_fraction_seed': 2,\n",
    "#         'feature_fraction': 0.9,\n",
    "#         'bagging_fraction': 0.8,\n",
    "#         'bagging_freq': 5,\n",
    "#         'min_data': 20,\n",
    "#         'min_hessian': 1,\n",
    "#         'verbose': -1,\n",
    "#         'silent': 0\n",
    "#         }\n",
    "params = {\n",
    "            'boosting_type': 'gbdt',\n",
    "            'objective': 'binary',\n",
    "            'num_leaves': 2 ** 7,\n",
    "            'metric': 'auc',\n",
    "            'min_child_weight': 5,\n",
    "            'learning_rate': 0.01,\n",
    "            'feature_fraction': 0.9,\n",
    "            'bagging_fraction': 0.9,\n",
    "            'seed': 2021,\n",
    "            'n_jobs':-1\n",
    "        }\n",
    "model = SBBTree(params=params,\n",
    "                stacking_num=5,\n",
    "                bagging_num=3,\n",
    "                bagging_test_size=0.33,\n",
    "                num_boost_round=10000,\n",
    "                early_stopping_rounds=200)\n",
    "\n",
    "model.fit(train, target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9efa0d9a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "\n",
    "# Split the dataset in two equal parts\n",
    "X_train, X_test, y_train, y_test = train_test_split(train, target, test_size=0.5, random_state=0)\n",
    "\n",
    "# model \n",
    "clf = RandomForestClassifier(n_jobs=-1)\n",
    "\n",
    "# Set the parameters by cross-validation\n",
    "\n",
    "tuned_parameters = {\n",
    "                    'n_estimators': [50, 100, 200]\n",
    "#                     ,'criterion': ['gini', 'entropy']\n",
    "#                     ,'max_depth': [2, 5]\n",
    "#                     ,'max_features': ['log2', 'sqrt', 'int']\n",
    "#                     ,'bootstrap': [True, False]\n",
    "#                     ,'warm_start': [True, False]\n",
    "                    }\n",
    "\n",
    "scores = ['precision']\n",
    "\n",
    "for score in scores:\n",
    "    print(\"# Tuning hyper-parameters for %s\" % score)\n",
    "    print()\n",
    "\n",
    "    clf = GridSearchCV(clf, tuned_parameters, cv=5,\n",
    "                       scoring='%s_macro' % score)\n",
    "    clf.fit(X_train, y_train)\n",
    "\n",
    "    print(\"Best parameters set found on development set:\")\n",
    "    print()\n",
    "    print(clf.best_params_)\n",
    "    print()\n",
    "    print(\"Grid scores on development set:\")\n",
    "    print()\n",
    "    means = clf.cv_results_['mean_test_score']\n",
    "    stds = clf.cv_results_['std_test_score']\n",
    "    for mean, std, params in zip(means, stds, clf.cv_results_['params']):\n",
    "        print(\"%0.3f (+/-%0.03f) for %r\"\n",
    "              % (mean, std * 2, params))\n",
    "    print()\n",
    "\n",
    "    print(\"Detailed classification report:\")\n",
    "    print()\n",
    "    print(\"The model is trained on the full development set.\")\n",
    "    print(\"The scores are computed on the full evaluation set.\")\n",
    "    print()\n",
    "    y_true, y_pred = y_test, clf.predict(X_test)\n",
    "    print(classification_report(y_true, y_pred))\n",
    "    print()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "37df7b9d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.2128"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred = model.predict(test)\n",
    "# df_out = pd.DataFrame()\n",
    "# df_out['user_id'] = test_data['user_id']\n",
    "# df_out['predict_prob'] = pred\n",
    "# df_out.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f301913d",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.sum([pred>0.207])/len(pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "9e7487f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "new_pred = []\n",
    "threshold = 0.207\n",
    "for index,x in enumerate(pred):\n",
    "    if x>threshold:\n",
    "        new_pred.append(1)\n",
    "    else:\n",
    "        new_pred.append(0)\n",
    "new_pred = np.array(new_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "256dee2b",
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Length of values (29894) does not match length of index (30000)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-33-b2e7efb0f09d>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0msubmit_data\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'loan_default'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnew_pred\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32mD:\\tools\\conda\\envs\\ml\\lib\\site-packages\\pandas\\core\\frame.py\u001b[0m in \u001b[0;36m__setitem__\u001b[1;34m(self, key, value)\u001b[0m\n\u001b[0;32m   3042\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3043\u001b[0m             \u001b[1;31m# set column\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 3044\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_set_item\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   3045\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3046\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_setitem_slice\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mslice\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\tools\\conda\\envs\\ml\\lib\\site-packages\\pandas\\core\\frame.py\u001b[0m in \u001b[0;36m_set_item\u001b[1;34m(self, key, value)\u001b[0m\n\u001b[0;32m   3118\u001b[0m         \"\"\"\n\u001b[0;32m   3119\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_ensure_valid_index\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 3120\u001b[1;33m         \u001b[0mvalue\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_sanitize_column\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   3121\u001b[0m         \u001b[0mNDFrame\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_set_item\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3122\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\tools\\conda\\envs\\ml\\lib\\site-packages\\pandas\\core\\frame.py\u001b[0m in \u001b[0;36m_sanitize_column\u001b[1;34m(self, key, value, broadcast)\u001b[0m\n\u001b[0;32m   3766\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3767\u001b[0m             \u001b[1;31m# turn me into an ndarray\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 3768\u001b[1;33m             \u001b[0mvalue\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msanitize_index\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   3769\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mndarray\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mIndex\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3770\u001b[0m                 \u001b[1;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m>\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\tools\\conda\\envs\\ml\\lib\\site-packages\\pandas\\core\\internals\\construction.py\u001b[0m in \u001b[0;36msanitize_index\u001b[1;34m(data, index)\u001b[0m\n\u001b[0;32m    746\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m!=\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mindex\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    747\u001b[0m         raise ValueError(\n\u001b[1;32m--> 748\u001b[1;33m             \u001b[1;34m\"Length of values \"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    749\u001b[0m             \u001b[1;34mf\"({len(data)}) \"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    750\u001b[0m             \u001b[1;34m\"does not match length of index \"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: Length of values (29894) does not match length of index (30000)"
     ]
    }
   ],
   "source": [
    "submit_data['loan_default'] = new_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "b04c1645",
   "metadata": {},
   "outputs": [],
   "source": [
    "submit_data.to_csv('lgb_stacking.csv',index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
